diff --git a/.github/workflows/e2e-fork.yaml b/.github/workflows/e2e-fork.yaml index df6ed1268..65b78230d 100644 --- a/.github/workflows/e2e-fork.yaml +++ b/.github/workflows/e2e-fork.yaml @@ -543,7 +543,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read @@ -771,21 +771,6 @@ jobs: name: cozyreport path: _out/cozyreport.tgz if-no-files-found: warn - # Both halves, and in this order: the compose project owns the nodes and - # the zpools outlive it. - - name: Tear down sandbox - if: always() - run: | - # Annotated, not fatal: same reasoning as the in-tree lane. The runner - # is discarded after the job, so a dirty host has no next tenant, and - # failing here would post the required "E2E Tests" status as a failure - # indistinguishable from a real one. - cleanup_rc=0 - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$? - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true - if [ "$cleanup_rc" -ne 0 ]; then - echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool" - fi # Flip the pending "E2E Tests" commit status posted by `resolve` to its # terminal state. Runs on failure too, so a failed publish or e2e turns the diff --git a/.github/workflows/e2e-tag.yaml b/.github/workflows/e2e-tag.yaml index f390077d0..d9bba8559 100644 --- a/.github/workflows/e2e-tag.yaml +++ b/.github/workflows/e2e-tag.yaml @@ -137,7 +137,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read checks: write # Breakpoint action updates its dedicated "Breakpoint Open" Check Run. @@ -292,7 +292,7 @@ jobs: # collect-report packages cozyreport plus per-failed-test crust-gather # snapshots captured before Chainsaw cleanup. # - # A ceiling of its own, not the job's 215 minutes. `Upload cozyreport.tgz` + # A ceiling of its own, not the job's 265 minutes. `Upload cozyreport.tgz` # is the next step and the collector writes its tarball at the very end of # its run, so a read that never returns spends what is left of the job # budget, the job is cancelled, and the upload never runs -- the artifact diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 495ccec38..57cce6790 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -250,7 +250,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read @@ -385,7 +385,7 @@ jobs: path: _out/chainsaw-report.xml if-no-files-found: ignore - # A ceiling of its own, not the job's 215 minutes. `Upload cozyreport.tgz` + # A ceiling of its own, not the job's 265 minutes. `Upload cozyreport.tgz` # is the next step and the collector writes its tarball at the very end of # its run, so a read that never returns spends what is left of the job # budget, the job is cancelled, and the upload never runs -- the artifact diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index b8b06fedf..24ee473ef 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -953,7 +953,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read @@ -1224,7 +1224,7 @@ jobs: # ▸ Collect debug information (always runs) # - # A ceiling of its own, not the job's 215 minutes. `Upload + # A ceiling of its own, not the job's 265 minutes. `Upload # cozyreport.tgz` is the next step and the collector writes its tarball # at the very end of its run, so a read that never returns spends what # is left of the job budget, the job is cancelled, and the upload never @@ -1235,7 +1235,7 @@ jobs: # 30 minutes is measured against the step: it takes 3-42s across sampled # runs, on failed e2e jobs as well as green ones. It RESERVES nothing -- # `timeout-minutes` caps a step, it does not carve budget out of the job, - # so a suite that has already burned 210 of the 215 leaves this step five + # so a suite that has already burned 260 of the 265 leaves this step five # minutes and the upload is cancelled with everything else. What the cap # buys is that a hung read cannot be the reason. # `continue-on-error` because the ceiling must not change what the job @@ -1348,28 +1348,6 @@ jobs: Resume from inside: `breakpoint resume`. Otherwise the breakpoint exits 10 minutes after the last SSH session disconnects. - # ▸ Tear down environment (always runs) - # - # Both halves, and in this order: the compose project owns the nodes and - # the zpools outlive it. A leftover pool makes the next run on a reused - # host adopt storage that already carries a dead cluster's volumes. - - name: Tear down sandbox - if: always() - run: | - # Loud, but never the job's verdict. A failed teardown means a dirty - # host, and this runner is discarded after the job, so there is no next - # run to protect -- while failing the job here posts the required - # "E2E Tests" status as a failure that no reader can tell apart from a - # real one. The annotation is what a reused host (a developer machine) - # needs; `delete-cluster-container` returning non-zero is not evidence - # about the product. - cleanup_rc=0 - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$? - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true - if [ "$cleanup_rc" -ne 0 ]; then - echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool" - fi - # Report the required "E2E Tests" gate as a commit status rather than relying # on a job named "E2E Tests". A required *job* that is skipped counts # as PASSED by branch protection — desired for docs-only PRs, but the exact diff --git a/.gitignore b/.gitignore index e265b9fd4..f1b99efdb 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ _repos /flux-shard-operator /kubeovn-plunger /lineage-controller-webhook +/observability-proxy /securitygroup-controller # User-specific stuff diff --git a/Makefile b/Makefile index 5a553db08..2feea00e9 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ build: build-deps make -C packages/system/cozystack-api image make -C packages/system/cozystack-controller image make -C packages/system/backup-controller image + make -C packages/system/observability-proxy image make -C packages/system/backupstrategy-controller image make -C packages/system/lineage-controller-webhook image make -C packages/system/migration-controller image diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index f0415f398..4c0d80ab8 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -58,7 +58,7 @@ type ConfigSpec struct { // Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret. // +kubebuilder:default:={} Users map[string]User `json:"users,omitempty"` - // Databases configuration map. + // Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. // +kubebuilder:default:={} Databases map[string]Database `json:"databases,omitempty"` // Backup configuration. diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 95c800aae..e457f4cac 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -74,6 +74,9 @@ type ConfigSpec struct { } type Disk struct { + // Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. + // +kubebuilder:validation:Minimum=1 + BootOrder *int `json:"bootOrder,omitempty"` // Disk bus type (e.g. "sata"). Bus string `json:"bus,omitempty"` // Disk name. diff --git a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go index 060281042..84f7315ac 100644 --- a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go @@ -61,7 +61,9 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { if in.Disks != nil { in, out := &in.Disks, &out.Disks *out = make([]Disk, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Networks != nil { in, out := &in.Networks, &out.Networks @@ -100,6 +102,11 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Disk) DeepCopyInto(out *Disk) { *out = *in + if in.BootOrder != nil { + in, out := &in.BootOrder, &out.BootOrder + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disk. diff --git a/api/backups/strategy/v1alpha1/kubernetes_types.go b/api/backups/strategy/v1alpha1/kubernetes_types.go new file mode 100644 index 000000000..704fc5d9f --- /dev/null +++ b/api/backups/strategy/v1alpha1/kubernetes_types.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// Package v1alpha1 defines strategy.backups.cozystack.io API types. +// +// Group: strategy.backups.cozystack.io +// Version: v1alpha1 +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, + &Kubernetes{}, + &KubernetesList{}, + ) + return nil + }) +} + +const ( + KubernetesStrategyKind = "Kubernetes" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster + +// Kubernetes defines a backup strategy for a tenant Kubernetes application. +// A tenant cluster keeps its state in two places the platform can reach: the +// cluster's key prefix in the tenant's shared etcd DataStore, and host-side +// objects (control-plane PKI Secrets, the application HelmReleases, and the +// DataVolumes that back the tenant's PersistentVolumes). The strategy captures +// the first with a one-shot Job and the second with Velero. The two are taken +// one after another, so a backup is crash-consistent at best. +type Kubernetes struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KubernetesSpec `json:"spec,omitempty"` + Status KubernetesStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// KubernetesList contains a list of Kubernetes backup strategies. +type KubernetesList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Kubernetes `json:"items"` +} + +// KubernetesSpec specifies the desired behavior of a Kubernetes backup strategy. +type KubernetesSpec struct { + // Template holds a PodTemplateSpec for the Job that dumps the cluster's + // key prefix from its etcd DataStore to object storage, writes it back on + // restore, and deletes the object on cleanup. The Job runs in the + // application namespace. Helm-like Go templates are supported in every + // string field and are rendered against: + // - `.Release.Name` and `.Release.Namespace` — the application's name and + // namespace; `.Release.Prefix` is `kubernetes-`. + // - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + // - `.ObjectKey` — the object-storage key of this backup's dump. + // - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + // - `.Datastore.Endpoints` — comma-separated etcd client URLs. + // - `.Datastore.CertificateSecret` — the Secret holding the cluster's + // etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + // - `.Parameters` — the resolved BackupClass strategy parameters. + // + // Stored schemaless for the same CRD-size reason as redis_types.go. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Template corev1.PodTemplateSpec `json:"template"` + + // Velero holds the Velero Backup and Restore specs for the host-side + // objects and volumes. Rendered against `.Application` and `.Parameters`, + // like the Velero strategy. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Velero VeleroTemplate `json:"velero"` + + // CertificateWarningThreshold is how close to expiry a control-plane + // certificate may be before a backup records a warning. Defaults to 720h. + // +optional + CertificateWarningThreshold *metav1.Duration `json:"certificateWarningThreshold,omitempty"` +} + +type KubernetesStatus struct { + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go index 3ad54f0fe..9efd1fb9b 100644 --- a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go @@ -839,6 +839,109 @@ func (in *KafkaStatus) DeepCopy() *KafkaStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes. +func (in *Kubernetes) DeepCopy() *Kubernetes { + if in == nil { + return nil + } + out := new(Kubernetes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Kubernetes) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesList) DeepCopyInto(out *KubernetesList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Kubernetes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesList. +func (in *KubernetesList) DeepCopy() *KubernetesList { + if in == nil { + return nil + } + out := new(KubernetesList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubernetesList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + in.Velero.DeepCopyInto(&out.Velero) + if in.CertificateWarningThreshold != nil { + in, out := &in.CertificateWarningThreshold, &out.CertificateWarningThreshold + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. +func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { + if in == nil { + return nil + } + out := new(KubernetesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus. +func (in *KubernetesStatus) DeepCopy() *KubernetesStatus { + if in == nil { + return nil + } + out := new(KubernetesStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MariaDB) DeepCopyInto(out *MariaDB) { *out = *in diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md index 33ec9ee89..81afd0ef6 100644 --- a/cmd/backport-batch/README.md +++ b/cmd/backport-batch/README.md @@ -9,16 +9,19 @@ go build ./cmd/backport-batch ./backport-batch run --line release-1.7 [--onto SHA] [--date YYYY-MM-DD] [--no-workflow-files] [--push] [--policy REF] ./backport-batch run --line release-1.7 --continue [--no-workflow-files] [--push] [--policy REF] ./backport-batch run --line release-1.7 --defer 4321[,4322] --reason "fails the e2e of the app" [--push] [--policy REF] +./backport-batch run --line release-1.7 --drop 4321[,4322] --reason "builds on the 1.8 storage rework" [--push] [--policy REF] +./backport-batch run --line release-1.7 --undrop 4321[,4322] [--push] [--policy REF] ./backport-batch check --line release-1.7 [--cut] [--policy REF] ./backport-batch fold --line release-1.7 --original 4321 --commit SHA [--policy REF] ./backport-batch verify --line release-1.7 [--head SHA] ./backport-batch notify --line release-1.7 [--mention org/team|login] [--push] +./backport-batch report --pr 5246 [--id ID] [--json] ./backport-batch lines [--policy REF] ``` [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) runs `lines`, then `run --continue --push`, `check` and `notify` for each line; [`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs it on schedule as the cozystack-ci App, with `--no-workflow-files`, which it hands to `run`. -Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Every command but `notify` fetches `origin` first; `notify` reads only GitHub. The batch runs only for the lines `lines` prints. +Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Every command but `notify` and `report` fetches `origin` first; `notify` reads only GitHub, and `report` reads GitHub and, for a report that keeps drops, the clone's history. The batch runs only for the lines `lines` prints. `run`, `check`, `fold` and `lines` read the `backport-policy` block of the policy page from `origin/main`, or from the ref `--policy` names, which is how a change to the page is tried before it merges; `verify` and `notify` do not read it and take no `--policy`. A line the block lists takes nothing merged into `main` before the day it gives, unless the original carries `backport/include`; any other line starts at its fork point. A page without exactly one readable block stops the run with exit 2. @@ -28,38 +31,58 @@ Build the binary wherever an exit code matters: `go run` turns any non-zero exit Switches the current working tree to `backport-batch/-`, created at `origin/`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it. -The pull request body goes to stdout, built from the line's own `.github/PULL_REQUEST_TEMPLATE.md`: the originals on the branch, each with how many commits it brought and which of them differ from the commit they were picked from by `git patch-id`, a resolution or a pick git had to fit, the deferred conflicts with the commit that failed and its files, the skipped pull requests counted by rule with the lists collapsed, the originals' release notes, and a `backport-batch` block of JSON that lists the deferred conflicts for `check`. The title is `fix(backport): batch for , `. A body that would pass GitHub's 65536-byte limit leaves out its longest skipped lists, and the run prints them on stderr instead. +The pull request body goes to stdout, built from the line's own `.github/PULL_REQUEST_TEMPLATE.md`: the originals on the branch, each with how many commits it brought and which of them differ from the commit they were picked from by `git patch-id`, a resolution or a pick git had to fit, the deferred conflicts with the commit that failed and its files, what is parked and why, the skipped pull requests counted by rule with the lists collapsed, the originals' release notes, and two blocks of JSON. An original whose unit differs from it, a pick whose patch-id differs from its commit's or a unit whose net patch-id differs from the original merge's, as when a first commit came out empty because the line already had that change, keeps its release note out of the release-note block: it goes under "Check these notes", with a `git range-diff --creation-factor=999 ^! ^!` for each pick that differs, which pairs the two however much the pick lost, and the `git diff` of the original's merge and of its unit, and the record and the report mark it `changed`. The two blocks of JSON are the [line's record](#the-lines-record-and-reports) in `backport-batch`, which later runs, `check` and `notify` act on, and the run's report in `backport-batch-report`. The title is `fix(backport): batch for , `. A body that would pass GitHub's 65536-byte limit leaves out its longest skipped lists, and the run prints them on stderr instead; one that what it has to carry still puts over the limit is refused before anything is pushed. `--onto SHA` continues a batch already pushed instead of starting a new one. SHA has to be the head the remote has for a batch branch of the line right now, read with `git ls-remote`, so a head someone pushed onto since is refused. What that branch carries stays, and only what it does not carry yet is picked on top, so the next push is a fast-forward. `--continue` takes SHA from the open batch pull request, and starts a new batch when none is open, which is what the scheduled run does. `--no-workflow-files` is for a token that may not push a change to `.github/workflows/`, which the scheduled run's App token may not. An original that changes anything there is not picked: the body lists it under "Waiting for a maintainer" with those files, and the `backport-batch` block records it with `"workflows": true`, so `check`, `notify` and a run that continues the batch know it waits. It is never a conflict. A run without the flag, a maintainer's with their own token, picks it like any other original, onto the open batch when it continues one. -`--defer N[,N] --reason TEXT` takes originals out of the open batch pull request, usually the ones a failing check was pinned on. It builds the batch branch again off `origin/` without their units: while the batch still sits on the line's head, the units before the first one it drops stay as they are, and every later one, a pushed resolution included, is picked again in its own shape, a merge's picks one by one and then the merge with its message. When dropping one leaves a later commit that no longer applies, the run stops and names that commit's original, to defer as well. It adds no new original. It also takes out an original that got `backport/skip` after the batch picked it, which a run never removes on its own and the body lists as still on the branch. The body records each deferred original with the reason in the `backport-batch` block; a run that continues this batch keeps them out, and the next batch tries them again. A batch that carries a merge commit other than an original's, as GitHub's update-branch button makes, is refused, since picking such a merge again would lose what it resolved. +A clean pick can still carry only part of its original: git deletes a file the line never had without a word, and leaves alone a file the line lacks. So every path the original modifies or deletes, `M` or `D` in `git diff --name-status --no-renames M^1 M`, is judged at its counterpart on the line: the path itself, or the one a rename `git diff -M` finds between the line's fork point and `M^1` on `main`, between the fork point and the line, or both in turn. A modified path counts as left without effect when its counterpart exists neither before nor after the pick; when the pick leaves an existing file as it was, the file counts as there only when its blob is the original's, binary files included, or when `git merge-file` of the original's preimage, the line's file and the original's postimage merges cleanly back into the line's file. A deleted path counts as left without effect when its counterpart is absent before a pick that changes something, a deletion the line already made being no change of that pick, or still there after it, whatever the pick changed in it; when the original adds that path itself, as when the line made the same rename, it is judged as a modified file against what the original makes of that path. Picks that change nothing carry the original only when the line already has what it does to every file it modifies or deletes, so a change made where its branch merged `main`, which no pick brings, is still missing. What cannot be judged, a mode or type change, content `merge-file` refuses, or a rename git skipped looking for, is shown with the original and never counts against it. An original whose net merge onto the line, `git merge-tree --merge-base=M^1`, leaves something without effect is not tried, and one whose picks do is taken back off the branch: either is parked as partial, listed under "Parked" with those paths, and its release note stays out of the body. A conflict is never called partial. `backport/include` lifts no park. -`--push` pushes the branch, refusing anything but a fast-forward of what the remote has, then opens the draft pull request against the line with `gh pr create --draft`, or updates the open batch pull request with `gh pr edit`. When the branch it refuses to push over has no pull request open on it, as a run that stopped between its push and opening the pull request leaves it, the refusal names the `run --onto --push` that continues it. While a batch pull request is open on another branch it refuses, so a line never has two. It never pushes to or edits a batch pull request whose body has no `backport-batch` block either: someone wrote that body by hand, and the batch is theirs to merge or close. After `--defer` alone it replaces the branch, with `--force-with-lease=refs/heads/:`, where head is what `git ls-remote` reads for the branch and has to be the head the rebuild started from: a commit pushed since is never dropped, by the tool's check or, if one lands between that check and the push, by the remote. +Before it tries an original, `run` looks at what the original [builds on](#what-an-original-builds-on) that the branch lacks, in merge order, so what became of each prerequisite in this run is known. A prerequisite the line or the open batch carries, or that this run picked or found empty, counts as there; once earlier picks moved the branch, the original is judged at the branch's head, so a line another unit brought counts too. Of what is left, a direct push, an original rules 2 to 4, the line's start, `backport/skip` or a drop keep out, and one that is itself parked never come: the original is parked, not tried, and listed under "Parked" with what it builds on. In the scheduled mode an original that changes a workflow, whose missing dependencies that never come are only tests or docs and whose workflow lines build on nothing missing, waits for a maintainer instead, with a note saying so. Prerequisites still to come, a conflict, a deferral, a wait for a maintainer, one rule 6 holds, make the original wait: it is listed under "Waiting on another original", recorded with `needs`, the prerequisites of the kind that decides, in the order not on the line, waiting for a maintainer, deferred, rule 6, and tried again by a later run. A reason given with `--defer` comes before all of this, and a partial park before any of it. + +`--defer N[,N] --reason TEXT` takes originals out of the open batch pull request, usually the ones a failing check was pinned on. It builds the batch branch again off `origin/` without their units: while the batch still sits on the line's head, the units before the first one it drops stay as they are, and every later one, a pushed resolution included, is picked again in its own shape, a merge's picks one by one and then the merge with its message. When dropping one leaves a later commit that no longer applies, the run stops and names that commit's original, to defer as well. It adds no new original. It also takes out an original that got `backport/skip` after the batch picked it, which a run never removes on its own and the body lists as still on the branch. The body records each deferred original with the reason in the `backport-batch` block; a run that continues this batch keeps them out, and the next batch tries them again. A batch that carries a merge commit other than an original's, as GitHub's update-branch button makes, is refused, since picking such a merge again would lose what it resolved. When it takes out everything the batch carries, there is nothing left to push, and `--push` closes the batch pull request instead, as [the line's record](#the-lines-record-and-reports) describes. + +`--drop N[,N] --reason TEXT` drops originals from the line: where `--defer` holds one out of this batch, a drop keeps it out of every later batch of the line too, until `--undrop N[,N]` withdraws it. A drop is recorded in the [line's record](#the-lines-record-and-reports), so both need an open batch pull request that carries one, and exit 2 without it; once a batch merges, its drops are carried into the next batch's record. A dropped original the open batch carries is taken off it as `--defer` would, and the rebuild is pushed the same way; any other drop or undrop changes only the record, so the batch stays as it is even when the line moved since. An undropped original is missing until the next run tries it. A drop names the original's merge commit on `main` as well as its number, and a record whose drop does not match what that merge commit merged stops every command with exit 2. An original the line already carries cannot be dropped, and neither can a number that merged into `main` before the line branched. The order is the rules', then the drop: what `backport/skip`, rules 2 to 4 or the line itself decide comes first, with the drop kept on record beside it, and `backport/include` does not lift a drop. The body lists dropped originals under their own heading among the skipped ones, `check` prints a note for each, and `fold` refuses one. + +`--push` pushes the branch, refusing anything but a fast-forward of what the remote has, then opens the draft pull request against the line with `gh pr create --draft`, or updates the open batch pull request with `gh pr edit`, and prints its URL on stderr. On the open batch it first posts the run's report as a comment, so a body never names a report nobody can read; a new batch carries the report in the body it opens with, and the comment follows. Before every write to the pull request it reads the pull request again, and stops with exit 2 once it is merged or closed. When the branch it refuses to push over has no pull request open on it, as a run that stopped between its push and opening the pull request leaves it, the refusal names the `run --onto --push` that continues it. While a batch pull request is open on another branch it refuses, so a line never has two. It never pushes to or edits a batch pull request whose body has no `backport-batch` block either: someone wrote that body by hand, and the batch is theirs to merge or close. After `--defer` alone it replaces the branch, with `--force-with-lease=refs/heads/:`, where head is what `git ls-remote` reads for the branch and has to be the head the rebuild started from: a commit pushed since is never dropped, by the tool's check or, if one lands between that check and the push, by the remote. A record that says the batch is being closed is acted on before anything else, and only by `run --push`. ### check -Exits 1 and prints the URL of every pull request the rules take that is neither on the line nor on the open batch pull request, 0 when there is none, and 2 when it cannot tell. A pull request that the open batch lists as deferred, a conflict or one taken out with `--defer`, prints as a warning and does not fail the check: deferring is a maintainer's call. So does one it lists as waiting for a maintainer because it changes workflows. +Exits 1 and prints the URL of every pull request the rules take that is neither on the line nor on the open batch pull request, 0 when there is none, and 2 when it cannot tell. A pull request that the [line's record](#the-lines-record-and-reports) lists as deferred, a conflict or one taken out with `--defer`, prints as a warning and does not fail the check: deferring is a maintainer's call. That holds once the batch that recorded it merged or closed too. So does one the open batch lists as waiting for a maintainer because it changes workflows; once that batch is gone, nothing lists it any more, and it is missing. One a maintainer dropped from the line prints as a note, `#N is dropped from on #M ()`. -`--cut` makes it the gate before a patch release or an rc is cut from a line. A cut tags the line, so a pull request that is only on the open batch pull request fails it too, with one line each: `#N is only on open batch PR #M, merge it before the cut`. One that waits there for a maintainer fails it as well, since nobody decided to leave it out: bring it over, or take it out with `run --defer` and a reason. Everything else is as without it. The scheduled run checks without `--cut`, since its batch is open by design. +`--cut` makes it the gate before a patch release or an rc is cut from a line. A cut tags the line, so a pull request that is only on the open batch pull request fails it too, with one line each: `#N is only on open batch PR #M, merge it before the cut`. One that waits there for a maintainer fails it as well, since nobody decided to leave it out, also when it is all the batch has left: bring it over, or take it out with `run --defer` and a reason. Everything else is as without it, and a cut that passes says `no change the policy takes is missing before the cut`. The scheduled run checks without `--cut`, since its batch is open by design. + +An original with `backport/skip` that the open batch picked before the label came prints a warning in either mode, naming the batch and `run --defer`, which takes it out; it fails neither. + +Every other original the policy takes that neither the line nor the open batch carries is decided in merge order, as [`run`](#run) decides it, with the open batch's found picks counting as there and the mode the open record saved, the scheduled run's when no batch is open, whose workflow waits on a closed record wait on nothing any more, so the original is judged as the scheduled run judges it: deferred for a reason, a warning; parked, a note naming what it builds on; in the scheduled mode a workflow original whose tests alone build on what never comes, waiting as the open batch lists it, or missing with none; and one that waits on other originals by the prerequisite that decides it: missing, which fails both modes, waiting for a maintainer, which warns and fails `--cut`, deferred, which warns, or held by rule 6, a note. A wait on one that waits itself inherits what decides that one. + +A unit on the open batch that leaves part of its original without effect, judged as [`run`](#run) judges a pick, at the unit's own first parent and whatever its original's rule, label or drop, prints a warning naming what it leaves out, and fails `--cut` ahead of any skip, drop or stale warning: take it out with `run --defer`. An original that is parked as partial prints as a note, `#N is parked: only part of it applies`, and fails neither. That includes one the open record parks as partial that the net merge at the line's head finds whole, since `run`'s picker found it after the picks before it: `check` and the operations that only change the record keep it parked while its net merge onto what the batch carries now still leaves something without effect, and call it missing once that went, as when the pick that made it partial was taken off. A park only the commit-by-commit replay sees, which no net merge shows, reads as missing in `check` until the next run. ### fold -Checks a hand-made resolution of a deferred conflict before it is pushed onto the batch, in the shape the run gives an original. For a merged pull request `--commit` is a merge whose first parent is the head of the open batch pull request and whose message names the original in exactly one `Backported-from:` trailer; its second parent ends a straight line from that head of picks of the pull request's own commits, in their order, each ending in the `(cherry picked from commit …)` line of the commit it came from, none claiming another commit of the same original, a commit of another pull request never among them. For a squash-merged one it is a single commit on that head whose `(cherry picked from commit …)` line names the original's commit on `main`. Either has to change something, and only files the original changes. The [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) gives the commands. fold prints `ok` and exits 0, or prints why not and exits 1. It never commits or pushes; push one checked resolution at a time, since the next one has to sit on the new head. +Checks a hand-made resolution of a deferred conflict before it is pushed onto the batch, in the shape the run gives an original. For a merged pull request `--commit` is a merge whose first parent is the head of the open batch pull request and whose message names the original in exactly one `Backported-from:` trailer; its second parent ends a straight line from that head of picks of the pull request's own commits, in their order, each ending in the `(cherry picked from commit …)` line of the commit it came from, none claiming another commit of the same original, a commit of another pull request never among them. For a squash-merged one it is a single commit on that head whose `(cherry picked from commit …)` line names the original's commit on `main`. An original parked for what it builds on folds like any other when the resolution is complete. Either has to change something, and only files the original changes or the paths a rename on `main` or on the line gave them, and has to leave nothing the original modifies or deletes without effect, judged as [`run`](#run) judges a pick: a resolution that ports what the line has and leaves out what it lacks is refused, since such a port goes to the line as a pull request of its own. The [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) gives the commands. fold prints `ok` and exits 0, or prints why not and exits 1. It never commits or pushes; push one checked resolution at a time, since the next one has to sit on the new head. ### verify -Holds every unit of a batch to the shape fold checks for one, the clean picks as well as resolutions: it walks the batch's first-parent line from `origin/` to `--head`, or to the head of the open batch pull request, and requires each unit to be one original, merged into `main` after the line branched, in that shape, and no original twice. A commit that carries no original fails it, and so does a merge of the line into the batch. It prints one line per unit, and exits 0 when all pass, 1 when one does not, and 2 when it cannot tell. With `--head` it reads git alone, nothing from GitHub. fold runs only where someone resolved a conflict and chose to run it; [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml) runs verify on every push to a batch pull request, from a build of `main`, with the batch head fetched as data. +Holds every unit of a batch to the shape fold checks for one, the clean picks as well as resolutions: it walks the batch's first-parent line from `origin/` to `--head`, or to the head of the open batch pull request, and requires each unit to be one original, merged into `main` after the line branched, in that shape, and no original twice. A commit that carries no original fails it, and so does a merge of the line into the batch, and a unit that leaves part of its original without effect, judged as [`run`](#run) judges a pick. It prints one line per unit, and exits 0 when all pass, 1 when one does not, and 2 when it cannot tell. With `--head` it reads git alone, nothing from GitHub. fold runs only where someone resolved a conflict and chose to run it; [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml) runs verify on every push to a batch pull request, from a build of `main`, with the batch head fetched as data. ### notify -Reads the open batch pull request of the line, trusted as described below, and prints one line for each thing that waits on it: a batch open for three days or more, the shortest gap between two scheduled runs, so one the previous run already saw; the conflicts it deferred; and the originals waiting for a maintainer because they change workflows. Nothing open prints nothing. +Reads the [line's record](#the-lines-record-and-reports) first, as the other commands do, and exits 2 when it cannot read it or when it says the open batch is being closed, with a batch open or not. Then it reads the open batch pull request of the line, trusted as described below, and prints one line for each thing that waits on it: a batch open for three days or more, the shortest gap between two scheduled runs, so one the previous run already saw; the conflicts it deferred; and the originals waiting for a maintainer because they change workflows. Nothing open prints nothing. + +An original the record has wait on others is listed once, as `#M, after #F`, where what decides its wait is listed: after a deferred conflict with the conflicts, after a workflow wait with the waits; one waiting on what is missing or held by rule 6 is not listed. A unit on the batch that carries only part of its original is listed apart, to be taken out before the cut. + +Each park the record of the open batch holds is named once. `run` marks a park `announced` when the line's record it found, open or closed, holds the same park announced: the same merge, kind, prerequisites, commits and files; a park that changed is news again. `notify` lists the parks not yet marked, and with `--push`, once the comment is written, marks them in the record with `gh pr edit --body-file`, only while the pull request is open and its body still carries the record `notify` read. A mark that fails after the comment exits 2, and the next `notify` names those parks once more. With no batch open nothing is named, and the next batch's first `notify` names what it parks. With `--push` it writes the same into one comment on the batch pull request: the conflicts, each to be resolved as the [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. The comment starts with `` and is one per pull request: a later run edits it in place rather than adding another, and once nothing waits it says so. Only a comment whose author the batch trusts counts as it, so one someone else wrote with the same first line is neither edited nor believed. It mentions `--mention`, an `org/team` or a login, with or without `@`, only while something waits. A failed batch CI is not among what it reports: the run has no trigger on the batch's CI finishing. +### report + +Prints a report a batch pull request keeps: the one its body carries, or the one `--id` names. It reads the report comments the batch trusts and falls back to the body, recomputes the report's ID from what it says, and exits 2 when a trusted comment does not match its ID, when a report says anything a run does not write or is for another line than the pull request's, or when the pull request keeps no such report. A report comment counts only when its author is one the batch trusts with a record, so anyone else's comment, whatever ID it carries, is not read. It prints what the classification was made of, whether the pull request's head or body shows the report was published, which it calls unconfirmed when neither does, and one line per original with its category and what the report recorded about it; `--json` prints the report as stored. A report whose comment the run never got to post is printed from the body, with a note that the next `run --push` posts it, or, on a merged or closed batch, that it is kept in the body only. It writes nothing. It reads GitHub, and for a report that keeps drops also the clone's history, without fetching: each drop is bound to `main` as the record's drops are, over the heads of `main` and the line the report was made from, so a drop of an original the report classified has to name that original's merge, and any other a merge of its number on `main`'s first-parent line. A clone without those heads exits 2, asking for a fetch. + ### lines Prints the release lines the batch runs for, newest first: of the `release-X.Y` branches on `origin`, the newest that have a stable `vX.Y.Z` tag there, as many as `released-lines` in the policy block says, two when it says nothing, and every branch newer than the newest of them. The tags are read with `git ls-remote`; a tag with a pre-release suffix does not count. `run`, `check` and `fold` refuse any other line, and rule 6 holds each line to the next newer one among them. @@ -94,10 +117,33 @@ Only the line's own commits count, the ones after the point where it branched fr Titles and bodies are read from the merge commits on `main`, which hold them as a maintainer merged them; the live pull request stays editable by its author. +## What an original builds on + +An original the policy takes can build on code its line does not have: another pull request merged into `main` after the line branched that the line never got, or a commit pushed to `main` directly. For each original the policy takes that neither the line nor the open batch carries, `run` reads what it builds on from git alone, in two ways: + +- A file it modifies or deletes whose counterpart, as [`run`](#run) follows renames, the line does not have is named by the first-parent commit on `main` that added it since the line branched: `git --literal-pathspecs log --first-parent --diff-merges=first-parent --no-renames --diff-filter=A -1 ..M^1 -- `. A file nothing added since the fork point is shown with the original, since no other pull request brought it. +- A line it removes or changes in a file the line has is given by `git blame --porcelain -w`, at `M^1` named by its full commit, to a commit, which counts through the first-parent commit on `main` that brought it: each commit of `main`'s first-parent line since the fork point brings itself and every commit of the range reachable from its other parents that no earlier one brought. A commit from before the fork point, which the line has, and the original's own count for nothing. The commit counts only while one of its lines, with its edges trimmed and nothing else, is absent from the line's version of the file, so a hand backport under another commit clears it. A file the line already has as the original makes it, the same blob or a clean `git merge-file` back into the line's file, builds on nothing. + +Lines a tool wrote are not blamed: a generated file whole, `zz_generated*`, one with a `Code generated … DO NOT EDIT` marker in its first 100 lines, `go.sum` and the package-manager lock files, and in an ApplicationDefinition under `packages/system/*-rd/cozyrds/` the keys `hack/update-crd.sh` writes, found by their indentation. Controller-gen CRDs and `values.schema.json` are blamed like any other file. Each `git log`, `git diff` and `git blame` the detector and the rename detection run has two minutes, and rename detection that runs out of time counts as skipped, and an original has at most 256 files, missing or blamed, 20 000 blamed lines and 8 MiB per file read for it. Evidence it cannot read, past one of those bounds, a binary file, a symlink, a submodule, a commit blame names that did not come through `main`, a path that is not UTF-8, or a rename git skipped looking for, is shown with the original, whatever becomes of it, under its line in the body and in the report's `diag`. Output of git the tool cannot read whole stops it with exit 2, and is never taken for an absent dependency. A dependency carried only by lines an original adds is not seen. `run` prints how long reading the line took on stderr. What follows from a dependency, a park or a wait, is in [`run`](#run) and [`check`](#check). + +## The line's record and reports + +The line's record is the `backport-batch` block of its newest batch pull request: the newest pull request against the line from a `backport-batch/-` branch of this repository, opened by an author the batch trusts as above, whether it is open, merged or closed. `run`, `check`, `fold` and `notify` find it in GitHub's list of the line's pull requests, newest first, reading at most ten pages of 100. A batch pull request a person opened without the block is passed over. A block whose author the batch does not trust, a bot's batch without its block, a block that cannot be read whole, a closed record newer than the open batch, or ten pages with no record stop the command with exit 2, naming the pull request, rather than fall back to an older record. + +The record is written only while its pull request is open: a merged or closed batch pull request is never edited or commented on again. What it recorded keeps counting, so a conflict or a deferral a merged batch recorded still prints as a warning. A workflow original it listed as waiting is missing, since no open batch lists it. A run that picks nothing while no batch is open has nowhere to write: it prints the body, says that its classification is not retained, and exits 0. + +The record is only as trustworthy as the body it sits in. Anyone with write access can edit any body, the bot's included, and neither the bot's authorship nor anything in the block proves who edited it last. A hand edit that leaves a readable record is believed; one that breaks it stops every command with exit 2, naming the pull request. A reason or a file name in the record counts only in the form a run writes it, one line with no control, zero-width or bidi character, so text someone typed into the body never reaches a terminal, a body or a comment as they typed it. When the newest batch pull request is deleted and no later one exists, nothing shows it was there: the older record governs, and what was recorded since is lost. + +The record holds only what later commands act on. Version 3, which this tool writes, adds what is parked and why, with the merge commit each park is bound to, the units on the batch that do only part of their original, the picks that differ from theirs, and what each wait on another original waits on; it reads versions 1 to 3, and a tool that reads only 1 and 2 refuses version 3 with exit 2. Every run that classifies the line also makes a report, one line of JSON in a `backport-batch-report` block: every original merged into `main` since the line branched in exactly one category, picked, missing, deferred with what it was deferred for, or skipped with the rule that skipped it, each drop the record keeps with the merge commit it is bound to, its reason and the batch pull request that recorded it, the commits on `main` that merged no pull request, the labels, and what the classification was made of: the heads of `main`, the line and the newer line, the policy, the batch head it describes, the tool's commit, the Actions run and the flags. Its ID is a hash of all that but the time and the run, so two runs that find the same make the same report, and it names no pull request, so a new batch's report is whole before GitHub numbers the pull request. The body carries the latest report beside the record, and a comment on the open batch pull request keeps each one: `` and the same block, folded away. A report already posted is not posted again, and the tool never edits or deletes such a comment, so a report the next run replaces in the body is still read back by `report`, also after the batch merged. A comment that looks like one but whose author the batch does not trust is not read. When a run stops after writing the body but before posting its comment, the next `run --push` on the open batch posts that report first, byte for byte; once the batch is closed nothing is written to it, and the report stays in its body. A run that picks nothing while no batch is open keeps no report. + +A `--defer` that leaves the batch carrying nothing closes it, and writes everything while the pull request is still open: the report comment, then the body, whose record keeps the deferrals and says the batch is closing, with its number, branch and the head the rebuild started from. It then reads the pull request and the branch again, closes the pull request while both are at that head, and deletes the branch with `--force-with-lease` on it, printing `closed `. The next batch tries what was deferred again. A close that stops part way is finished by the next `run --push`, before it does anything else: it closes and deletes when the pull request is open at the frozen head, deletes the branch when the pull request is closed and the branch still at that head, and goes on as usual once both are gone. A run that finishes a close stops there, with exit 0, or with exit 2 when it was asked for `--defer`, `--drop` or `--undrop`, which it does not apply. It never takes a head it did not freeze: with the branch moved, it stops with exit 2 and names both heads. `run` without `--push`, `check` and `fold` stop with exit 2 while the pull request is still open; once it is closed they go on, warning about a branch the close left, and `notify` says nothing. A closed pull request is never written to, so its record says it was closing for good, and a closed pull request whose branch is gone counts as closed. + +One way for the branch to move is a new batch of the same day: it reuses the branch name, and if its push succeeds and opening its pull request fails, the closed batch is still the newest record while its branch now holds the new batch. The next run stops on it, naming the branch, and so does `--onto` with its head. Check what it carries before deleting it with `git push origin --delete backport-batch/-`: the next run picks again what the tool put there from the line, but not a commit someone pushed onto it, such as a resolved conflict. Separate writes to GitHub are not atomic: the re-reads and the leases refuse a race they see, but cannot prevent one. + ## What CI verifies The tool does not build or test anything. CI on the batch pull request runs the line's own build and tests against what was picked, and `verify` against its shape, and a maintainer reviews and merges it. A deferred conflict waits for the next batch unless someone resolves it, checks the resolution with `fold` and pushes it, and then CI runs again. ## Untrusted text -Titles, release notes and file names in the body come from whoever opened the originals. A title, a reason or a file name goes into a code span whose fence no backtick inside can close, and the release notes go into the `release-note` block with every run of three backticks or tildes defused, so GitHub acts on no mention, reference, link or HTML in them. Inside that block, in case a note is copied out of it, links lose any target outside this repository, and closing keywords, cross-repository references and mentions have their `@`, `#` and `://` swapped for fullwidth signs, which act on nothing however the backticks around them pair up. Zero-width, bidi and control characters are dropped first, and everything the tool prints passes the same filter, so nothing it relays can drive a terminal. Commit messages are read through `git cat-file --batch`, which gives each object's length, and end at their first NUL as `git log` shows them, so a message cannot pass for a second commit or hide a trailer from a reader of the log. Every value handed to git comes after `--end-of-options` or `--`, refs pass `git check-ref-format`, SHAs have to be 40 hex digits, and nothing runs through a shell. +Titles, release notes and file names in the body come from whoever opened the originals, and a reason from whoever ran `--defer`. A title, a reason or a file name goes into a code span whose fence no backtick inside can close, and the release notes go into the `release-note` block with every run of three backticks or tildes defused, so GitHub acts on no mention, reference, link or HTML in them. Inside that block, in case a note is copied out of it, links lose any target outside this repository, and closing keywords, cross-repository references and mentions have their `@`, `#` and `://` swapped for fullwidth signs, which act on nothing however the backticks around them pair up. The record and the report are one line of JSON each, whose encoding leaves no newline, `<`, `>` or `&` in them, so nothing in a file name or a reason ends their block or opens HTML, and a report comment names no one: numbers are bare, and its only text outside the block is the line, the date and the report's ID. Zero-width, bidi and control characters are dropped first, and everything the tool prints passes the same filter, so nothing it relays can drive a terminal. Commit messages are read through `git cat-file --batch`, which gives each object's length, and end at their first NUL as `git log` shows them, so a message cannot pass for a second commit or hide a trailer from a reader of the log. Every value handed to git comes after `--end-of-options` or `--`, refs pass `git check-ref-format`, SHAs have to be 40 hex digits, and nothing runs through a shell. diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go index ef26aace8..3c025517e 100644 --- a/cmd/backport-batch/assemble.go +++ b/cmd/backport-batch/assemble.go @@ -33,6 +33,8 @@ type pick struct { files []string // failed is the commit whose pick conflicted. failed string + // needs are the originals still to come it waits on. + needs []int // more counts the conflicting files a recorded conflict no longer names. more int reason string @@ -75,8 +77,10 @@ func (a *app) run(args []string) int { cont := fs.Bool("continue", false, "continue the open batch pull request, or start a batch when none is open") dateArg := fs.String("date", "", "batch date, YYYY-MM-DD (default: today, UTC)") deferArg := fs.String("defer", "", "originals to take out of the open batch, N[,N]; rebuilds it without them") + dropArg := fs.String("drop", "", "originals to drop from the line, N[,N]; recorded on the open batch, and taken off it") + undropArg := fs.String("undrop", "", "originals whose drop to withdraw, N[,N]") noWorkflows := fs.Bool("no-workflow-files", false, "leave an original that changes "+workflowsDir+" to a maintainer, for a token that cannot push one") - reasonArg := fs.String("reason", "", "why --defer takes them out, recorded in the batch body") + reasonArg := fs.String("reason", "", "why --defer takes them out or --drop drops them, recorded in the batch body") push := fs.Bool("push", false, "push the branch and open or update the draft batch PR") if err := parse(fs, args); err != nil { return 2 @@ -96,9 +100,30 @@ func (a *app) run(args []string) int { return a.fail(fmt.Errorf("--onto: %w", err)) } } - var deferred []int - reason := "" + var deferred, drops []int + reason, ops := "", 0 + for _, op := range []string{*deferArg, *dropArg, *undropArg} { + if op != "" { + ops++ + } + } switch { + case ops > 1: + return a.fail(errors.New("--defer, --drop and --undrop each change the open batch pull request on their own; give one")) + case *dropArg != "" || *undropArg != "": + if *onto != "" || *cont || *dateArg != "" { + return a.fail(errors.New("--drop and --undrop change the open batch pull request; each takes no --onto, --continue or --date")) + } + if *undropArg != "" && *reasonArg != "" { + return a.fail(errors.New("--undrop takes no --reason")) + } + list := *dropArg + *undropArg + if drops, err = parseNumbers(map[bool]string{true: "--drop", false: "--undrop"}[*dropArg != ""], list); err != nil { + return a.fail(err) + } + if reason = capped(oneLine(clean(*reasonArg, false)), titleLimit); *dropArg != "" && reason == "" { + return a.fail(errors.New("--drop needs a --reason, which the line's record keeps")) + } case *deferArg != "" || *reasonArg != "": if deferred, reason, err = parseDefer(*deferArg, *reasonArg); err != nil { return a.fail(err) @@ -115,10 +140,23 @@ func (a *app) run(args []string) int { if err := fetch(); err != nil { return a.fail(err) } + started := time.Now() b, err := a.prepare(l) if err != nil { return a.fail(err) } + if goOn, code := a.resumeClose(b, *push, deferred != nil || drops != nil); !goOn { + return code + } + if err := b.analyze(); err != nil { + return a.fail(err) + } + // The scheduled job has half an hour for every line; this says where it + // goes. + fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond)) + if drops != nil { + return a.redrop(b, drops, reason, *undropArg != "", *push) + } if deferred != nil { return a.rebuild(b, deferred, reason, *push) } @@ -147,10 +185,8 @@ func (a *app) run(args []string) int { // Continuing the open batch keeps out what was deferred from it by hand; // a new batch tries those again. held := map[int]deferredEntry{} - if carried != nil && b.pr != nil && b.pr.Head.Ref == branch { - if held, err = parseDeferred(b.pr.Body, l); err != nil { - return a.fail(fmt.Errorf("batch pull request #%d: %w", b.pr.Number, err)) - } + if carried != nil && b.src.open && b.pr.Head.Ref == branch { + held = b.src.rec.deferred() } if err := a.warnReset(branch, start); err != nil { return a.fail(err) @@ -160,24 +196,72 @@ func (a *app) run(args []string) int { } r := newReport(b) + r.opts = runOptions{NoWorkflowFiles: *noWorkflows, Mode: modeNew} + // A drop is a decision about the line, not about one batch: a new batch + // keeps the ones the closed batch recorded. + r.dropped = b.src.rec.dropped() if carried != nil { + r.opts.Mode = modeContinue if r.stale, err = keptOut(b, carried); err != nil { return a.fail(err) } } + // cls is what each original became in this run, to an original that + // builds on it; the rules decide the rest. + mode := analysisMode{noWorkflowFiles: *noWorkflows} + cls := map[*verdict]edgeClass{} + classOf := func(f *verdict) edgeClass { + if c, ok := b.ruleClass(f); ok { + return c + } + if c, ok := cls[f]; ok { + return c + } + return edgeMissing + } for _, v := range b.qualifying() { + on := false + if carried != nil { + if on, err = b.onBranch(carried, v); err != nil { + return a.fail(err) + } + } switch { - case carried != nil && found(carried, v): + case on: r.onBranch = append(r.onBranch, v) + cls[v] = edgeSatisfied continue case held[v.o.number].Reason != "": r.held = append(r.held, pick{v: v, status: "held", reason: held[v.o.number].Reason}) + cls[v] = edgeDeferred continue case v.onBatch != "": v.rule, v.reason = ruleOnLine, fmt.Sprintf("on the open batch pull request #%d", b.pr.Number) r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) continue } + if v.partial != nil { + // Judged at the line's head; what the batch picked since may + // supply what it lacked there. + if err := a.partialAtHead(b, v); err != nil { + return a.fail(err) + } + if v.partial != nil { + r.parked = append(r.parked, v) + cls[v] = edgeNever + continue + } + } + if len(v.deps) > 0 { + kind, err := a.dependsAt(b, r, v, mode, classOf) + if err != nil { + return a.fail(err) + } + if kind != edgeSatisfied { + cls[v] = kind + continue + } + } if *noWorkflows { files, err := workflowFiles(v.commits) if err != nil { @@ -185,18 +269,30 @@ func (a *app) run(args []string) int { } if len(files) > 0 { r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: files}) + cls[v] = edgeWorkflow continue } } - p, err := pickOriginal(v) + p, err := pickOriginal(v, func(base, head string) ([]string, error) { + omitted, diag, err := b.omittedEffects(v, base, head) + v.addDiag(diag...) + return omitted, err + }) if err != nil { return a.fail(err) } switch p.status { + case "partial": + v.partial = p.files + v.park = &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkPartial, Files: sortedClean(p.files)} + r.parked = append(r.parked, v) + cls[v] = edgeNever case "picked": r.onBranch = append(r.onBranch, v) + cls[v] = edgeSatisfied case "conflict": r.deferred = append(r.deferred, p) + cls[v] = edgeDeferred case "empty": v.rule, v.reason = ruleOnLine, "the earlier picks already carry it" r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) @@ -214,22 +310,83 @@ func (a *app) run(args []string) int { if !descends { return a.fail(fmt.Errorf("%.12s does not descend from --onto %.12s", head, *onto)) } + if err := b.markPartial(r, carried.head); err != nil { + return a.fail(err) + } } return a.finish(b, r, branch, date, *push, false) } -// keptOut lists the originals with backport/skip that the batch picked before -// the label came: a run never takes a commit off the batch, --defer does. +// dependsAt decides, from what v builds on that the branch lacks, whether +// the run leaves it out: waiting for a maintainer in the scheduled mode +// when only tests or docs build on what never comes, parked when anything +// else never comes, or waiting on prerequisites still to come. What is +// there, the line's, the batch's, or what this run picked or found empty, +// counts for nothing; once earlier picks moved the branch, v is judged at +// its head. It returns what v became to an original that builds on it, +// edgeSatisfied when the run goes on to pick it. +func (a *app) dependsAt(b *batch, r *report, v *verdict, mode analysisMode, classOf func(*verdict) edgeClass) (edgeClass, error) { + deps := v.deps + head, err := resolve("HEAD") + if err != nil { + return 0, err + } + if head != b.lineHead { + if deps, err = b.depsAt(v, head); err != nil { + return 0, err + } + } + never, pending := edges(deps, classOf) + primary, err := workflowPrimary(mode, v, never, pending) + switch { + case err != nil: + return 0, err + case primary: + files, err := workflowFiles(v.commits) + if err != nil { + return 0, err + } + v.addDiag(testsOnly(v, never, b.line)) + r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: files}) + return edgeWorkflow, nil + case len(never) > 0: + v.park = dependencyPark(v, never) + r.parked = append(r.parked, v) + return edgeNever, nil + case len(pending) > 0: + class, needs := strongest(pending) + r.needs = append(r.needs, pick{v: v, status: "waiting", needs: numbers(needs)}) + return class, nil + } + return edgeSatisfied, nil +} + +// partialAtHead judges v again where the run has got to, when earlier picks +// moved the branch past the line's head. +func (a *app) partialAtHead(b *batch, v *verdict) error { + head, err := resolve("HEAD") + if err != nil || head == b.lineHead { + return err + } + return b.partialAt(v, head) +} + +// keptOut lists the originals with backport/skip, or dropped, that the batch +// still carries: a run never takes a commit off the batch, --defer does. func keptOut(b *batch, carried *history) ([]*verdict, error) { var out []*verdict for _, v := range b.verdicts { - if v.label != labelSkip { + if v.rule != ruleSkipLabel && v.rule != ruleDropped { continue } if err := b.load(v); err != nil { return nil, err } - if found(carried, v) { + on, err := b.onBranch(carried, v) + if err != nil { + return nil, err + } + if on { out = append(out, v) } } @@ -244,7 +401,7 @@ func found(h *history, v *verdict) bool { // newReport starts the report of a run with what the rules left out and what // the labels overrode. func newReport(b *batch) *report { - r := &report{line: b.line, newer: b.newer, skipped: map[int][]*verdict{}, other: len(b.other), start: b.start} + r := &report{line: b.line, newer: b.newer, skipped: map[int][]*verdict{}, other: len(b.other), start: b.start, carried: b.src.rec} for _, v := range b.verdicts { if v.rule != 0 { r.skipped[v.rule] = append(r.skipped[v.rule], v) @@ -266,12 +423,32 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r if r.shapes, err = summarize(b, head, r.onBranch); err != nil { return a.fail(err) } - title := prTitle(b.line, date.Format(time.DateOnly)) + if rebuilt && head == b.lineHead && b.pr != nil { + // Deferring everything the batch carries leaves nothing to push: the + // batch is closed, and its record says so first, with the head the + // close is leased on. + r.closing = &closing{PR: b.pr.Number, Branch: branch, Head: b.prHead} + r.opts.Mode = modeClose + } + r.built = builtFrom{Main: b.mainHead, Line: b.lineHead, Newer: b.newerHead, Policy: b.policyAt, Batch: head, Tool: a.tool, + Run: runURL(os.LookupEnv), At: a.now().UTC().Format(time.RFC3339), Options: r.opts} + doc, err := buildReport(b, r) + if err != nil { + return a.fail(err) + } + p := publication{branch: branch, head: head, title: prTitle(b.line, date.Format(time.DateOnly)), report: storedReport{doc: doc, raw: reportJSON(doc)}} + p.record = r.lineRecord(doc) + // A record the next command would refuse is never written. + if err := p.record.validate(b.line); err != nil { + return a.fail(fmt.Errorf("the record this run made %w", err)) + } + r.blocks = recordBlock(p.record) + "\n\n" + reportBlock(p.report.raw) template, err := lineTemplate(b.lineHead) if err != nil { return a.fail(err) } body, left := r.body(template) + p.body = body fmt.Fprint(a.stdout, body) skipped := 0 for _, items := range r.skipped { @@ -279,29 +456,44 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r } for _, rule := range left { for _, v := range r.skipped[rule] { - fmt.Fprintf(a.stderr, "skipped by rule %d: %s %s (%s)\n", rule, v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) + fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) } } for _, p := range r.waiting { fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) } - fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\n", - branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, title) + fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", + branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID) if !push { return 0 } if head == b.lineHead && b.pr == nil { // Picking nothing is not a failure of the run; check still reports - // whatever the line lacks. + // whatever the line lacks. Merged and closed batches are never + // written, so there is nowhere to keep what the run found. fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) + fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) return 0 } - if err := a.publish(b, branch, head, title, body, rebuilt); err != nil { + if err := a.publish(b, p, rebuilt); err != nil { return a.fail(err) } return 0 } +// lineRecord is the record a run leaves for the line: what later commands +// act on, with the full classification left to the report it names. +func (r *report) lineRecord(doc reportDoc) lineRecord { + built := doc.Built + dropped := slices.Clone(r.dropped) + if dropped == nil { + dropped = []droppedEntry{} + } + slices.SortFunc(dropped, func(a, b droppedEntry) int { return a.Original - b.Original }) + return lineRecord{Version: recordVersion, Line: doc.Line, Built: &built, Report: doc.ID, Closing: r.closing, Deferred: slices.Clone(doc.Primary.Deferred), Dropped: dropped, Stale: slices.Clone(doc.Overlays.Stale), + Parked: carryAnnounced(doc.Primary.Parked, r.carried), Partial: slices.Clone(doc.Overlays.Partial), Changed: slices.Clone(doc.Overlays.Changed)} +} + // branchDate is the date a batch branch was opened under. func branchDate(branch string) (time.Time, error) { m := batchBranchRE.FindStringSubmatch(branch) @@ -446,80 +638,312 @@ func lineTemplate(lineHead string) (string, error) { return git("show", "--end-of-options", lineHead+":"+template) } +// publication is what a run publishes: the branch at head, and the body that +// carries the record and the report. +type publication struct { + branch, head, title, body string + report storedReport + record lineRecord +} + // publish pushes the batch branch, then opens the draft batch pull request or // updates the open one. A run pushes only a fast-forward of what the remote // has. A rebuild replaces the branch, and only the head it was rebuilt from: // the lease is that head, as ls-remote reads it now, so a commit pushed since // is never dropped. -func (a *app) publish(b *batch, branch, head, title, body string, rebuilt bool) error { - if b.pr != nil && b.pr.Head.Ref != branch { +// +// The report goes up as a comment before the push and the body that names +// it, so no write leaves a body naming a report nobody can read; a new batch +// carries the report in the body it is opened with, and the comment follows. +// Everything is rendered and measured before the first write. +func (a *app) publish(b *batch, p publication, rebuilt bool) error { + if b.pr != nil && b.pr.Head.Ref != p.branch { return fmt.Errorf("the open batch pull request #%d is on %s; continue it with --onto %s", b.pr.Number, b.pr.Head.Ref, b.prHead) } // A body without the block was written by hand, and the tool's would // replace it whole, title included; such a batch is the maintainer's. - if b.pr != nil && !strings.Contains("\n"+strings.ReplaceAll(b.pr.Body, "\r\n", "\n")+"\n", "\n"+deferredFence+"\n") { + if b.pr != nil && !b.src.open { return fmt.Errorf("batch pull request #%d has no %s block, so backport-batch did not write it; merge or close it, and the next run opens a batch of its own", b.pr.Number, deferredFence) } - if head == b.lineHead { - if rebuilt { - return fmt.Errorf("without what is deferred, %s carries nothing beyond %s; close batch pull request #%d instead", branch, b.line.branch(), b.pr.Number) + if p.head == b.lineHead { + if rebuilt && p.record.Closing != nil { + return a.closeBatch(b, p) } - return fmt.Errorf("%s carries nothing beyond %s; nothing to push", branch, b.line.branch()) + return fmt.Errorf("%s carries nothing beyond %s; nothing to push", p.branch, b.line.branch()) } heads, err := remoteBatchHeads(b.line) if err != nil { return err } - pushed, onRemote := heads[branch] + pushed, onRemote := heads[p.branch] lease := "" switch { case rebuilt: if !onRemote || pushed != b.prHead { - return fmt.Errorf("%s on %s is at %.12s, not at %.12s, which the rebuild started from; run again", branch, remote, pushed, b.prHead) + return fmt.Errorf("%s on %s is at %.12s, not at %.12s, which the rebuild started from; run again", p.branch, remote, pushed, b.prHead) } lease = pushed - case onRemote && pushed != head: + case onRemote && pushed != p.head: have, err := lookup(pushed) if err != nil { return err } descends := false if have != "" { - if descends, err = isAncestor(pushed, head); err != nil { + if descends, err = isAncestor(pushed, p.head); err != nil { return err } } if !descends { - refusal := fmt.Sprintf("%s on %s is at %.12s, which %.12s does not descend from; refusing a non-fast-forward push", branch, remote, pushed, head) + refusal := fmt.Sprintf("%s on %s is at %.12s, which %.12s does not descend from; refusing a non-fast-forward push", p.branch, remote, pushed, p.head) // A run that stopped between its push and opening the pull request // leaves the branch, and the next run that day builds it anew. - if b.pr == nil { + switch c := b.src.carrier; { + case b.pr != nil: + case c != nil && !b.src.open && c.Head.Ref == p.branch: + // Continuing a batch that merged or closed would publish + // what nobody can merge any more. + refusal += fmt.Sprintf("; it is the branch of batch pull request #%d, which is %s; the next run picks again what the tool put on it, but not a commit someone pushed there, such as a resolved conflict: check what %.12s carries, then delete it with git push origin --delete %s and run again", c.Number, prState(c), pushed, p.branch) + default: refusal += fmt.Sprintf("; no batch pull request is open on it, and backport-batch run --line %s --onto %s --push continues it and opens one", b.line.branch(), pushed) } return errors.New(refusal) } } - if pushed != head { - if _, err := git(pushArgs(branch, head, lease)...); err != nil { + if err := p.fits(); err != nil { + return err + } + if b.pr != nil { + if err := a.keepReports(b, &p); err != nil { return err } } - f, err := os.CreateTemp("", "backport-batch-body-*.md") + if pushed != p.head { + if _, err := git(pushArgs(p.branch, p.head, lease)...); err != nil { + return err + } + } + name, err := writeTemp("backport-batch-body-*.md", p.body) if err != nil { return err } - defer os.Remove(f.Name()) - if _, err := f.WriteString(body); err != nil { - f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } + defer os.Remove(name) if b.pr == nil { - return a.gh.createPull(b.line.branch(), branch, title, f.Name()) + n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name) + if err != nil { + return err + } + fmt.Fprintf(a.stderr, "opened draft %s\n", url) + if err := a.postReport(n, p.report); err != nil { + return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err) + } + return nil } - return a.gh.editPull(b.pr.Number, title, f.Name()) + if err := a.stillOpen(b.pr.Number); err != nil { + return err + } + if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil { + return err + } + fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) + return nil +} + +// fits says why GitHub would not take the comment or the body. +func (p *publication) fits() error { + if _, err := reportComment(p.report); err != nil { + return err + } + if len(p.body) > bodyLimit { + return fmt.Errorf("the body is %d bytes with only what it cannot leave out, over GitHub's %d; nothing is published", len(p.body), bodyLimit) + } + return nil +} + +// keepReports makes sure the open batch's comments keep both the report its +// body carries now and the one this run made, in that order, each once. +// When this run's report is one already kept, the body carries the kept +// bytes, which differ only in when and in which job it was made. +func (a *app) keepReports(b *batch, p *publication) error { + pr := b.pr + posted, bad, err := findReports(a.gh, pr.Number, b.line) + if err != nil { + return err + } + for _, why := range bad { + a.warn("batch pull request #%d: %s; it does not count as a report", pr.Number, why) + } + earlier, err := bodyReport(pr.Body, b.line) + if err != nil { + return fmt.Errorf("batch pull request #%d: %w", pr.Number, err) + } + kept, ok := posted[p.report.doc.ID] + if !ok && earlier != nil && earlier.doc.ID == p.report.doc.ID { + kept, ok = *earlier, true + } + if ok { + p.body = strings.Replace(p.body, reportBlock(p.report.raw), reportBlock(kept.raw), 1) + p.report = kept + if err := p.fits(); err != nil { + return err + } + } + if earlier != nil { + if _, ok := posted[earlier.doc.ID]; !ok { + if err := a.postReport(pr.Number, *earlier); err != nil { + return fmt.Errorf("posting the earlier report %.12s on #%d: %w", earlier.doc.ID, pr.Number, err) + } + posted[earlier.doc.ID] = *earlier + } + } + if _, ok := posted[p.report.doc.ID]; !ok { + if err := a.postReport(pr.Number, p.report); err != nil { + return fmt.Errorf("posting report %.12s on #%d: %w", p.report.doc.ID, pr.Number, err) + } + } + return nil +} + +// pullURL is where a pull request is on GitHub. +func pullURL(pr *pullRequest) string { + if pullURLRE.MatchString(pr.HTMLURL) { + return pr.HTMLURL + } + return fmt.Sprintf("%spull/%d", ourRepoURL, pr.Number) +} + +// closeBatch closes the open batch that --defer left carrying nothing. Every +// write happens while it is open: the report, then the body whose record +// says it is closing and at which head. The close and the deletion of its +// branch then go only while both are still at that head. +func (a *app) closeBatch(b *batch, p publication) error { + c := *p.record.Closing + if err := p.fits(); err != nil { + return err + } + if err := a.keepReports(b, &p); err != nil { + return err + } + name, err := writeTemp("backport-batch-body-*.md", p.body) + if err != nil { + return err + } + defer os.Remove(name) + if err := a.stillOpen(c.PR); err != nil { + return err + } + if err := a.gh.editPull(c.PR, p.title, name); err != nil { + return fmt.Errorf("recording the close of #%d: %w", c.PR, err) + } + return a.finishClose(b.line, pullURL(b.pr), c) +} + +// finishClose closes the batch pull request a record says is closing, and +// deletes its branch, each only while it is at the head the record froze. +func (a *app) finishClose(l line, url string, c closing) error { + pr, err := a.gh.pull(c.PR) + if err != nil { + return fmt.Errorf("closing #%d, reading it again: %w", c.PR, err) + } + if st := prState(&pr); st != "open" || pr.Head.SHA != c.Head { + return fmt.Errorf("closing #%d: it is %s at %.12s, not open at %.12s, which its record froze", c.PR, st, terminalSafe(pr.Head.SHA), c.Head) + } + heads, err := remoteBatchHeads(l) + if err != nil { + return err + } + if h := heads[c.Branch]; h != c.Head { + return fmt.Errorf("closing #%d: %s on %s is at %.12s, not at %.12s, which its record froze", c.PR, c.Branch, remote, h, c.Head) + } + if err := a.gh.closePull(c.PR); err != nil { + return fmt.Errorf("closing #%d: %w; backport-batch run --line %s --push finishes it", c.PR, err, l.branch()) + } + if err := a.deleteBranch(l, c); err != nil { + return err + } + fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) + return nil +} + +// deleteBranch deletes the branch of a closed batch, leased on the head it +// was closed at: one someone pushed since is a person's to delete or keep. +func (a *app) deleteBranch(l line, c closing) error { + heads, err := remoteBatchHeads(l) + if err != nil { + return err + } + switch h, ok := heads[c.Branch]; { + case !ok: + return nil + case h != c.Head: + return movedBranch(c, h) + } + if _, err := git(deleteArgs(c.Branch, c.Head)...); err != nil { + return fmt.Errorf("deleting %s, left by the close of #%d: %w; backport-batch run --line %s --push deletes it", c.Branch, c.PR, err, l.branch()) + } + return nil +} + +// takenBranch is the refusal for a branch a close left that an open batch +// pull request now uses, whatever it holds, or nil. +func takenBranch(b *batch, c closing) error { + if b.pr == nil || b.pr.Head.Ref != c.Branch { + return nil + } + return fmt.Errorf("batch pull request #%d is open on %s, which the close of #%d left; the branch is that pull request's now, and the close leaves it alone", b.pr.Number, c.Branch, c.PR) +} + +func movedBranch(c closing, h string) error { + return fmt.Errorf("batch pull request #%d was closed at %.12s, but %s is at %.12s now; the next run picks again what the tool put on it, but not a commit someone pushed there, such as a resolved conflict: check what %.12s carries, then delete it with git push origin --delete %s, and run again", c.PR, c.Head, c.Branch, h, h, c.Branch) +} + +// resumeClose finishes a close the record says is under way, before a run +// does anything else, and only under --push. It says whether the run may go +// on: only once the batch is closed and its branch gone. A run that finishes +// one stops there, and fails when it was asked to change the batch. +func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) { + if b.src.rec == nil || b.src.rec.Closing == nil { + return true, 0 + } + c := *b.src.rec.Closing + heads, err := remoteBatchHeads(b.line) + if err != nil { + return false, a.fail(err) + } + h, onRemote := heads[c.Branch] + switch { + case b.src.open && h != c.Head: + return false, a.fail(fmt.Errorf("batch pull request #%d is being closed, but %s is at %.12s, not at %.12s, which its record froze; a person closes it, or pushes the branch back, by hand", c.PR, c.Branch, h, c.Head)) + case b.src.open && !push: + return false, a.fail(fmt.Errorf("batch pull request #%d is being closed: closing it and deleting %s are left; backport-batch run --line %s --push finishes it", c.PR, c.Branch, b.line.branch())) + case b.src.open: + if err := a.finishClose(b.line, pullURL(b.src.carrier), c); err != nil { + return false, a.fail(err) + } + return false, a.finished(c, changing) + case !onRemote: + return true, 0 + case takenBranch(b, c) != nil: + return false, a.fail(takenBranch(b, c)) + case h != c.Head: + return false, a.fail(movedBranch(c, h)) + case !push: + return false, a.fail(fmt.Errorf("batch pull request #%d was closed, but %s is still at %.12s; backport-batch run --line %s --push deletes it", c.PR, c.Branch, h, b.line.branch())) + } + if err := a.deleteBranch(b.line, c); err != nil { + return false, a.fail(err) + } + fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) + return false, a.finished(c, changing) +} + +// finished is the exit of a run that finished a close: a change it was asked +// for needs an open batch pull request, and none is left to make it on. +func (a *app) finished(c closing, changing bool) int { + if !changing { + return 0 + } + return a.fail(fmt.Errorf("finished the close of #%d and made no other change: what this run was asked to change needs an open batch pull request, which the next run opens", c.PR)) } // pushArgs pushes head to branch: as a fast-forward, which the remote checks, @@ -531,3 +955,8 @@ func pushArgs(branch, head, lease string) []string { } return append(args, "--end-of-options", remote, head+":refs/heads/"+branch) } + +// deleteArgs deletes branch, only while the remote has it at lease. +func deleteArgs(branch, lease string) []string { + return []string{"push", "--quiet", "--force-with-lease=refs/heads/" + branch + ":" + lease, "--end-of-options", remote, ":refs/heads/" + branch} +} diff --git a/cmd/backport-batch/assemble_test.go b/cmd/backport-batch/assemble_test.go index 513ae6ff2..31dfd7dda 100644 --- a/cmd/backport-batch/assemble_test.go +++ b/cmd/backport-batch/assemble_test.go @@ -258,9 +258,17 @@ func TestRunPush(t *testing.T) { f.lines() f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) gh := &fakeGH{} - if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 { + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { t.Fatalf("run --push = %d\n%s", code, stderr) } + if !strings.Contains(stderr, "opened draft https://github.com/cozystack/cozystack/pull/90\n") { + t.Errorf("the run does not print the pull request it opened:\n%s", stderr) + } + l, _ := parseLine("release-1.1") + if _, err := parseDeferred(stdout, l); err != nil || !strings.HasPrefix(stdout, "## What this PR does") || strings.Contains(stdout, "https://github.com/cozystack/cozystack/pull/90") { + t.Errorf("stdout is no longer the body alone (%v):\n%s", err, stdout) + } head := f.git("rev-parse", "HEAD") if got := f.remoteHead(batchBranch1); got != head { t.Errorf("remote %s = %q, want %s", batchBranch1, got, head) @@ -272,9 +280,13 @@ func TestRunPush(t *testing.T) { f.git("switch", "-q", "main") f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, gh.bodies[0])} - if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--onto", head, "--push"); code != 0 { + code, _, stderr = f.exec(gh, "run", "--line", "release-1.1", "--onto", head, "--push") + if code != 0 { t.Fatalf("run --onto --push = %d\n%s", code, stderr) } + if !strings.Contains(stderr, "updated https://github.com/cozystack/cozystack/pull/90\n") { + t.Errorf("the run does not print the pull request it updated:\n%s", stderr) + } if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): five (#5)", "fix(app): six (#6)"}) { t.Errorf("the pushed batch carries %q, want #5 then #6", got) } diff --git a/cmd/backport-batch/batchbot_test.go b/cmd/backport-batch/batchbot_test.go index 987a134c2..ff1cb342a 100644 --- a/cmd/backport-batch/batchbot_test.go +++ b/cmd/backport-batch/batchbot_test.go @@ -24,7 +24,8 @@ import ( // A maintainer's run by hand continues the batch pull request the scheduled // run opened as its bot, with nothing set: it edits that pull request rather -// than opening a second one for the line. +// than opening a second one for the line. A record another bot wrote is no +// record the batch trusts, and the run stops rather than reach past it. func TestHandRunContinuesTheBotsBatch(t *testing.T) { for _, tc := range []struct { name, author string @@ -44,12 +45,12 @@ func TestHandRunContinuesTheBotsBatch(t *testing.T) { f.now = f.now.AddDate(0, 0, 3) code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") - if code != 0 { + if tc.continued && code != 0 || !tc.continued && (code != 2 || !strings.Contains(stderr, "batch pull request #90 (open) carries a ```backport-batch block but was opened by \"renovate[bot]\"")) { t.Fatalf("run --continue = %d\n%s", code, stderr) } continued := slices.Equal(gh.edited, []string{"#90 fix(backport): batch for release-1.1, 2026-10-26"}) && len(gh.created) == 1 - if continued != tc.continued { - t.Errorf("edited %q, created %q; want the batch of %s continued=%v\n%s", gh.edited, gh.created, tc.author, tc.continued, stderr) + if continued != tc.continued || len(gh.created) != 1 { + t.Errorf("edited %q, created %q; want the batch of %s continued=%v and no second one\n%s", gh.edited, gh.created, tc.author, tc.continued, stderr) } if !tc.continued && !strings.Contains(stderr, "ignoring #90") { t.Errorf("the run does not say it ignores #90:\n%s", stderr) diff --git a/cmd/backport-batch/body.go b/cmd/backport-batch/body.go index a2d3a4c09..dbb9d6050 100644 --- a/cmd/backport-batch/body.go +++ b/cmd/backport-batch/body.go @@ -176,6 +176,9 @@ type deferredEntry struct { More int `json:"more,omitempty"` Reason string `json:"reason,omitempty"` Workflows bool `json:"workflows,omitempty"` + // Needs, from version 3, are the originals still to come that it waits + // on, those of the kind that decides the wait. + Needs []int `json:"needs,omitempty"` } // How many conflicting files of one original the body names, in the list and @@ -190,25 +193,11 @@ type deferredRecord struct { Deferred []deferredEntry `json:"deferred"` } -// deferredBlock is the machine-readable list of what the batch defers. One -// line of JSON: encoding/json escapes newlines and <, > and &, so nothing in a -// file name or a reason can end the fence or open HTML. It leaves bidi and -// zero-width characters as they are, so file names go through clean first, -// as the reason does. +// deferredBlock is a version 1 record, which batches carried before records +// had a version. encoding/json leaves bidi and zero-width characters as they +// are, so file names go through clean first, as the reason does. func deferredBlock(l line, conflicts, held, waiting []pick) string { - rec := deferredRecord{Line: l.branch(), Deferred: []deferredEntry{}} - for _, p := range conflicts { - files := cleanFiles(p.files[:min(len(p.files), filesKept)]) - rec.Deferred = append(rec.Deferred, deferredEntry{Original: p.v.o.number, Commit: p.failed, Files: files, More: len(p.files) - len(files) + p.more}) - } - for _, p := range held { - rec.Deferred = append(rec.Deferred, deferredEntry{Original: p.v.o.number, Files: []string{}, Reason: capped(oneLine(clean(p.reason, false)), titleLimit)}) - } - for _, p := range waiting { - files := cleanFiles(p.files[:min(len(p.files), filesKept)]) - rec.Deferred = append(rec.Deferred, deferredEntry{Original: p.v.o.number, Files: files, More: len(p.files) - len(files) + p.more, Workflows: true}) - } - data, _ := json.Marshal(rec) + data, _ := json.Marshal(deferredRecord{Line: l.branch(), Deferred: deferredEntries(conflicts, held, waiting, nil)}) return deferredFence + "\n" + string(data) + "\n```" } @@ -220,54 +209,13 @@ func cleanFiles(files []string) []string { return out } -// parseDeferred reads what a batch body defers: exactly one block, for this -// line, naming each original once. +// parseDeferred reads what a batch body's record defers. func parseDeferred(body string, l line) (map[int]deferredEntry, error) { - var blocks []string - lines := strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") - for i := 0; i < len(lines); i++ { - if strings.TrimSpace(lines[i]) != deferredFence { - continue - } - if i+2 >= len(lines) || strings.TrimSpace(lines[i+2]) != "```" { - return nil, fmt.Errorf("the %s block is not one line of JSON", deferredFence) - } - blocks = append(blocks, lines[i+1]) - i += 2 + rec, err := parseRecord(body, l) + if err != nil { + return nil, err } - switch len(blocks) { - case 0: - return map[int]deferredEntry{}, nil - case 1: - default: - return nil, fmt.Errorf("the body holds %d %s blocks", len(blocks), deferredFence) - } - dec := json.NewDecoder(strings.NewReader(blocks[0])) - dec.DisallowUnknownFields() - var rec deferredRecord - if err := dec.Decode(&rec); err != nil { - return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) - } - if rec.Line != l.branch() { - return nil, fmt.Errorf("the %s block is for %q, not %s", deferredFence, rec.Line, l.branch()) - } - out := map[int]deferredEntry{} - for _, d := range rec.Deferred { - if d.Original <= 0 { - return nil, fmt.Errorf("the %s block names original %d", deferredFence, d.Original) - } - if d.Commit != "" && checkSHA(d.Commit) != nil { - return nil, fmt.Errorf("the %s block names commit %q for #%d", deferredFence, d.Commit, d.Original) - } - if d.Workflows && (d.Commit != "" || d.Reason != "") { - return nil, fmt.Errorf("the %s block has #%d wait for a maintainer and defers it as well", deferredFence, d.Original) - } - if _, dup := out[d.Original]; dup { - return nil, fmt.Errorf("the %s block names #%d twice", deferredFence, d.Original) - } - out[d.Original] = d - } - return out, nil + return rec.deferred(), nil } // prTitle is built from the line and the date alone, never from carried text. @@ -276,7 +224,7 @@ func prTitle(l line, date string) string { } // skipRules are the rules a skipped original is reported under, in order. -var skipRules = []int{ruleBeforeStart, ruleOnLine, ruleType, ruleBreaking, ruleTouches, ruleNewer} +var skipRules = []int{ruleBeforeStart, ruleOnLine, ruleType, ruleBreaking, ruleTouches, ruleNewer, ruleDropped} // report is what one run put on the batch branch and what it left out. type report struct { @@ -286,6 +234,7 @@ type report struct { deferred []pick // conflicts held []pick // what --defer took out, with the reason waiting []pick // what changes workflows, left to a maintainer + needs []pick // what waits on other originals still to come skipped map[int][]*verdict other int // overrides are the originals a backport/ label kept out or let in, and @@ -293,8 +242,53 @@ type report struct { overrides []*verdict stale []*verdict start time.Time - // shapes says, for an original on the branch, what it brought. - shapes map[*verdict]string + // shapes says, for an original on the branch, what it brought, and + // partialFiles what a partial unit leaves without effect. + shapes map[*verdict]unitShape + partialFiles map[*verdict][]string + // missing are originals the policy takes that the run neither tried nor + // found anywhere. + missing []*verdict + // opts is how the run was asked to classify, and built what it read. + opts runOptions + built builtFrom + // blocks are the record and the report the body ends with. + blocks string + // closing is the close of the open batch this run leaves carrying + // nothing. + closing *closing + // dropped are the maintainers' drops the record keeps. + dropped []droppedEntry + // parked are what the batch leaves out because it cannot land whole, + // each with its park, and partial the originals on the branch whose unit + // does only part of what they do on main. + parked []*verdict + partial []*verdict + // carried is the line's record this run found, whose announcements the + // parks that did not change keep. + carried *lineRecord +} + +// ruleName is how output names what kept an original out: the policy's +// number for its own rules, and words for what the policy does not number. +func ruleName(rule int) string { + switch rule { + case ruleSkipLabel: + return labelSkip + case ruleBeforeStart: + return "rule 1, before the line's start" + case ruleDropped: + return "a maintainer's drop" + } + return "rule " + strconv.Itoa(rule) +} + +// listTitle titles the list of what a rule skipped, a drop among them. +func (r *report) listTitle(rule int) string { + if rule == ruleDropped { + return "Dropped from " + codeSpan(r.line.branch()) + " by a maintainer" + } + return r.ruleTitle(rule) } func (r *report) ruleTitle(rule int) string { @@ -319,16 +313,16 @@ func (r *report) ruleTitle(rule int) string { // in details written out. func (r *report) what(details map[int]bool) string { var b strings.Builder - fmt.Fprintf(&b, "Backports to %s from `main`, picked by the rules of the [backport policy](%s) in merge order. A merged pull request comes as its own commits, picked with `git cherry-pick -x` so each keeps its author, message and sign-offs, and a merge commit that carries the original's merge message with a `%s:` trailer naming it; a squash-merged one comes as one pick. `backport-batch verify` checks that shape. A pick listed as changed differs from the commit it came from; `git range-diff ^1..^2 ^1..^2` shows how. A conflict or a failing check here is a [backport task](%sREADME.md).\n\n", codeSpan(r.line.branch()), policyURL, batchTrailer, tasksURL) + fmt.Fprintf(&b, "Backports to %s from `main`, picked by the rules of the [backport policy](%s) in merge order. A merged pull request comes as its own commits, picked with `git cherry-pick -x` so each keeps its author, message and sign-offs, and a merge commit that carries the original's merge message with a `%s:` trailer naming it; a squash-merged one comes as one pick. `backport-batch verify` checks that shape. A pick listed as changed differs from the commit it came from, and its release note waits under Check these notes. A conflict or a failing check here is a [backport task](%sREADME.md).\n\n", codeSpan(r.line.branch()), policyURL, batchTrailer, tasksURL) if len(r.onBranch) == 0 { b.WriteString("Nothing is picked yet.\n") } for _, v := range r.onBranch { shape := "" - if s := r.shapes[v]; s != "" { - shape = ", " + s + if s, ok := r.shapes[v]; ok { + shape = ", " + s.String() } - fmt.Fprintf(&b, "- %s %s%s\n", prLink(v.o.number), titleSpan(v.o.title), shape) + fmt.Fprintf(&b, "- %s %s%s\n%s", prLink(v.o.number), titleSpan(v.o.title), shape, diagItems(v)) } if len(r.deferred) > 0 { fmt.Fprintf(&b, "\n### Deferred conflicts\n\nThese do not pick cleanly onto this branch. Each names the first of its commits that conflicted and the files it conflicted in, and nothing of it is on the branch. Each moves to the next batch, unless a resolution checked with `backport-batch fold` is pushed here first, as [resolving a conflict](%sresolve-conflict.md) describes.\n\n", tasksURL) @@ -344,13 +338,13 @@ func (r *report) what(details map[int]bool) string { if p.failed != "" { at = " at " + codeSpan(p.failed[:12]) } - fmt.Fprintf(&b, "- %s %s%s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), at, strings.Join(files, ", ")) + fmt.Fprintf(&b, "- %s %s%s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), at, strings.Join(files, ", "), diagItems(p.v)) } } if len(r.held) > 0 { b.WriteString("\n### Deferred to the next batch\n\nTaken out of this batch with `backport-batch run --defer`, each for the reason given. This pull request keeps them out; the next batch picks them again.\n\n") for _, p := range r.held { - fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), titleSpan(p.reason)) + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), titleSpan(p.reason), diagItems(p.v)) } } if len(r.waiting) > 0 { @@ -363,7 +357,41 @@ func (r *report) what(details map[int]bool) string { if more := len(p.files) - len(files) + p.more; more > 0 { files = append(files, fmt.Sprintf("%d more", more)) } - fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), strings.Join(files, ", ")) + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), strings.Join(files, ", "), diagItems(p.v)) + } + } + if changed := r.changed(); len(changed) > 0 { + b.WriteString("\n### Check these notes\n\nThese picks differ from their originals, so their release notes are left out of the release-note block until someone checks that each still holds for this line. Each comes with what to compare: every pick that differs against the commit it came from, and the original's change against the unit's.\n") + for _, v := range changed { + s := r.shapes[v] + fmt.Fprintf(&b, "\n%s %s, carried by %s:\n\n", prLink(v.o.number), titleSpan(v.o.title), codeSpan(s.unit[:12])) + if n := releaseNote(v.o.body); n != "" { + fmt.Fprintf(&b, "```text\n%s\n```\n\n", neutralNote(n)) + } else { + b.WriteString("It has no release note.\n\n") + } + for _, p := range s.pairs { + fmt.Fprintf(&b, "- %s\n", codeSpan(fmt.Sprintf("git range-diff --creation-factor=999 %s^! %s^!", p.source[:12], p.pick[:12]))) + } + fmt.Fprintf(&b, "- %s\n- %s\n", codeSpan(fmt.Sprintf("git diff %s^1 %s", v.o.merge[:12], v.o.merge[:12])), codeSpan(fmt.Sprintf("git diff %s^1 %s", s.unit[:12], s.unit[:12]))) + } + } + if len(r.partial) > 0 { + fmt.Fprintf(&b, "\n### Still on this branch: only part of it applies\n\nEach of these is on this branch but leaves files of its original as they were on %s, so it carries only part of what it does on `main`. Take each out with `backport-batch run --defer` before the cut; what applies of it goes as a pull request of its own.\n\n", codeSpan(r.line.branch())) + for _, v := range r.partial { + fmt.Fprintf(&b, "- %s %s: leaves %s without effect\n", prLink(v.o.number), titleSpan(v.o.title), fileSpans(r.partialFiles[v])) + } + } + if len(r.parked) > 0 { + fmt.Fprintf(&b, "\n### Parked\n\nThese cannot land on %s whole, so the batch does not pick them: each would leave part of it without effect there, or builds on a change that never reaches the line. A maintainer ports what applies as a pull request of its own, or drops the original with `backport-batch run --drop`; `backport/include` does not lift a park.\n\n", codeSpan(r.line.branch())) + for _, v := range r.parked { + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(v.o.number), titleSpan(v.o.title), parkReason(*v.park), diagItems(v)) + } + } + if len(r.needs) > 0 { + fmt.Fprintf(&b, "\n### Waiting on another original\n\nThese build on another original that has not reached %s yet, so the batch does not try them; each names the one it waits on, and the batch tries it again once that one is there.\n\n", codeSpan(r.line.branch())) + for _, p := range r.needs { + fmt.Fprintf(&b, "- %s %s: after %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), prLinks(p.needs), diagItems(p.v)) } } if len(r.overrides) > 0 { @@ -382,7 +410,7 @@ func (r *report) what(details map[int]bool) string { if !details[rule] { left = ", too many to list here; the run's output lists them" } - fmt.Fprintf(&b, "- %s: %d%s\n", r.ruleTitle(rule), len(items), left) + fmt.Fprintf(&b, "- %s: %d%s\n", r.listTitle(rule), len(items), left) } if r.other > 0 { fmt.Fprintf(&b, "- Commits on `main` that merged no pull request: %d\n", r.other) @@ -390,20 +418,24 @@ func (r *report) what(details map[int]bool) string { for _, rule := range skipRules { items := r.skipped[rule] if len(items) > 0 && details[rule] { - fmt.Fprintf(&b, "\n
%s\n\n", r.ruleTitle(rule)) + fmt.Fprintf(&b, "\n
%s\n\n", r.listTitle(rule)) for _, v := range items { fmt.Fprintf(&b, "- %s %s, %s\n", prLink(v.o.number), titleSpan(v.o.title), titleSpan(v.reason)) } b.WriteString("\n
\n") } } - b.WriteString("\n" + deferredBlock(r.line, r.deferred, r.held, r.waiting) + "\n") + b.WriteString("\n" + r.blocks + "\n") return b.String() } // outcome is what became of an original a label overrode. func (r *report) outcome(v *verdict) string { switch { + case slices.Contains(r.partial, v): + return "still on this branch, but only part of it applies; take it out with `backport-batch run --defer`" + case slices.Contains(r.stale, v) && v.rule == ruleDropped: + return "dropped, but still on this branch; take it out with `backport-batch run --defer`" case slices.Contains(r.stale, v): return "still on this branch, picked before the label; take it out with `backport-batch run --defer`" case v.label == labelSkip: @@ -416,15 +448,83 @@ func (r *report) outcome(v *verdict) string { return "deferred to the next batch" case slices.ContainsFunc(r.waiting, func(p pick) bool { return p.v == v }): return "waiting for a maintainer: it changes " + codeSpan(workflowsDir) + case slices.Contains(r.parked, v): + return "parked: " + parkReason(*v.park) + case slices.ContainsFunc(r.needs, func(p pick) bool { return p.v == v }): + i := slices.IndexFunc(r.needs, func(p pick) bool { return p.v == v }) + return "waiting on " + prLinks(r.needs[i].needs) case v.rule != 0: - return "skipped by rule " + strconv.Itoa(v.rule) + ", " + titleSpan(v.reason) + return "skipped by " + ruleName(v.rule) + ", " + titleSpan(v.reason) } return "not picked" } +// diagItems lists, under an original's line, what could not be judged about +// it or what it was seen to build on, each as a code span. +func diagItems(v *verdict) string { + var b strings.Builder + for _, d := range v.diag { + fmt.Fprintf(&b, " - %s\n", titleSpan(d)) + } + return b.String() +} + +// fileSpans lists files as code spans, the first few of them. +func fileSpans(files []string) string { + var out []string + for _, f := range files[:min(len(files), filesListed)] { + out = append(out, codeSpan(f)) + } + if more := len(files) - len(out); more > 0 { + out = append(out, fmt.Sprintf("%d more", more)) + } + return strings.Join(out, ", ") +} + +// prLinks links to pull requests by number. +func prLinks(numbers []int) string { + var out []string + for _, n := range numbers { + out = append(out, prLink(n)) + } + return strings.Join(out, ", ") +} + +// parkReason says why a park leaves its original out, for the body. +func parkReason(p parkedEntry) string { + if p.Kind == parkPartial { + return "only part of it applies; it would leave " + fileSpans(p.Files) + " without effect" + } + var needs []string + for _, n := range p.Needs { + needs = append(needs, prLink(n)) + } + for _, c := range p.Commits { + needs = append(needs, codeSpan(c[:12])) + } + return "builds on " + strings.Join(needs, ", ") + ", which never reaches this line" +} + +// changed lists the originals on the branch whose unit differs from them, in +// merge order. +func (r *report) changed() []*verdict { + var out []*verdict + for _, v := range r.onBranch { + if s, ok := r.shapes[v]; ok && s.changed() { + out = append(out, v) + } + } + return out +} + +// notes are the release notes of what the branch carries as its originals +// made it; a changed pick's note waits under Check these notes. func (r *report) notes() string { var notes []string for _, v := range r.onBranch { + if s, ok := r.shapes[v]; ok && s.changed() { + continue + } if n := releaseNote(v.o.body); n != "" { notes = append(notes, neutralNote(n)) } diff --git a/cmd/backport-batch/body_test.go b/cmd/backport-batch/body_test.go index 07f2f8582..3f746d17d 100644 --- a/cmd/backport-batch/body_test.go +++ b/cmd/backport-batch/body_test.go @@ -286,7 +286,7 @@ func TestBodyFollowsTheTemplate(t *testing.T) { l, _ := parseLine("release-1.6") template := "\n\n## What this PR does\n\n\n### Screenshots\n\n\n\n### Downstream repositories\n\n- [ ] No downstream repository is affected by this change\n\n### Release note\n\n```release-note\n\n```\n" o := &original{number: 7, title: "fix(x): y", body: "```release-note\nfix(x): note\n```"} - r := &report{line: l, onBranch: []*verdict{{o: o}}, skipped: map[int][]*verdict{}} + r := &report{line: l, onBranch: []*verdict{{o: o}}, skipped: map[int][]*verdict{}, blocks: deferredBlock(l, nil, nil, nil)} body, _ := r.body(template) for _, want := range []string{"## What this PR does\n\nBackports to", "- [#7](https://github.com/cozystack/cozystack/pull/7) `fix(x): y`", "### Downstream repositories\n\n- [ ] No downstream", "```release-note\nfix(x): note\n```", deferredFence} { if !strings.Contains(body, want) { @@ -305,7 +305,7 @@ func TestBodyFollowsTheTemplate(t *testing.T) { for i := range 40 { many = append(many, &verdict{o: &original{number: i + 1, title: "fix(x): y", body: "```release-note\n" + long + "\n```"}}) } - r = &report{line: l, onBranch: many, skipped: map[int][]*verdict{ruleType: many}} + r = &report{line: l, onBranch: many, skipped: map[int][]*verdict{ruleType: many}, blocks: deferredBlock(l, nil, nil, nil)} if body, _ := r.body(template); len(body) > bodyLimit || !strings.Contains(body, "truncated") || !strings.Contains(body, deferredFence) { t.Errorf("a body over the limit is %d bytes (limit %d), truncated=%v", len(body), bodyLimit, strings.Contains(body, "truncated")) } @@ -322,7 +322,7 @@ func TestBodyFollowsTheTemplate(t *testing.T) { for i := range 80 { conflict.files = append(conflict.files, fmt.Sprintf("chart/f%02d.yaml", i)) } - r = &report{line: l, onBranch: many[:1], deferred: []pick{conflict}, skipped: map[int][]*verdict{ruleType: lots, ruleOnLine: few}} + r = &report{line: l, onBranch: many[:1], deferred: []pick{conflict}, skipped: map[int][]*verdict{ruleType: lots, ruleOnLine: few}, blocks: deferredBlock(l, []pick{conflict}, nil, nil)} body, left := r.body(template) if !slices.Equal(left, []int{ruleType}) { t.Errorf("left out the lists of rules %v, want only rule 2", left) @@ -366,3 +366,17 @@ func TestBodyLinksTaskPagesThatExist(t *testing.T) { } } } + +// A body that what it has to carry pushes over GitHub's limit is refused +// before anything is written, the replay of an earlier report included, +// although the report alone would fit a comment. +func TestBodyRefusesMandatoryContentOverTheLimit(t *testing.T) { + f, gh, head := bigWaits(t, 4) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push") + if code != 2 || !strings.Contains(stderr, "with only what it cannot leave out, over GitHub's 65536") { + t.Errorf("run --push = %d, want a refusal of the body:\n%s", code, stderr) + } + if f.remoteHead(batchBranch1) != head || len(gh.issueComments) != 0 || len(gh.edited) != 0 { + t.Errorf("the refused run moved the batch to %.12s, wrote %d comments and edited %q", f.remoteHead(batchBranch1), len(gh.issueComments), gh.edited) + } +} diff --git a/cmd/backport-batch/check.go b/cmd/backport-batch/check.go index 00e3d5f16..d8acf4e37 100644 --- a/cmd/backport-batch/check.go +++ b/cmd/backport-batch/check.go @@ -18,7 +18,9 @@ package main import ( "fmt" + "slices" "strconv" + "strings" ) // check says whether every original the policy takes is on the line, on the @@ -46,59 +48,242 @@ func (a *app) check(args []string) int { if err != nil { return a.fail(err) } - deferred := map[int]deferredEntry{} - if b.pr != nil { - if deferred, err = parseDeferred(b.pr.Body, l); err != nil { - return a.fail(fmt.Errorf("batch pull request #%d: %w", b.pr.Number, err)) + if err := a.warnLeftBranch(b); err != nil { + return a.fail(err) + } + // The scheduled run's mode, as the open batch saved it; with none open, + // the scheduled run's, which is what the next batch is built in. + mode := analysisMode{noWorkflowFiles: true} + if b.src.open { + mode.noWorkflowFiles = b.src.savedNoWorkflowFiles() + } + disp, err := b.dispositions(mode, *cut) + if err != nil { + return a.fail(err) + } + var missing, failing []string + for _, v := range b.verdicts { + d, ok := disp[v] + switch { + case !ok: + case d.kind == outcomeMissing: + missing = append(missing, d.text) + case d.kind == outcomeUnmerged || d.kind == outcomeWaiting: + failing = append(failing, d.text) + case d.kind == outcomeWarning: + a.warn("%s", d.text) + case d.kind == outcomeNote: + fmt.Fprintf(a.stderr, "note: %s\n", d.text) } } - var missing, unmerged, waiting []*verdict - for _, v := range b.qualifying() { - e, isDeferred := deferred[v.o.number] - switch { - case v.onBatch != "": - if *cut { - unmerged = append(unmerged, v) - } - case isDeferred && e.Workflows && *cut: - waiting = append(waiting, v) - case isDeferred && e.Workflows: - a.warn("waiting on #%d for a maintainer, since it changes %s: %s %s", b.pr.Number, workflowsDir, v.o.url(), terminalSafe(v.o.title)) - case isDeferred && e.Reason != "": - a.warn("deferred on #%d to the next batch: %s %s (%s)", b.pr.Number, v.o.url(), terminalSafe(v.o.title), terminalSafe(e.Reason)) - case isDeferred: - a.warn("deferred conflict on #%d: %s %s", b.pr.Number, v.o.url(), terminalSafe(v.o.title)) - default: - missing = append(missing, v) + if len(missing)+len(failing) == 0 { + if *cut { + fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch()) + } else { + fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) } + return 0 } where := "and no batch pull request is open" if b.pr != nil { where = "nor on batch pull request #" + strconv.Itoa(b.pr.Number) } - switch { - case len(missing) == 0 && len(unmerged) == 0 && len(waiting) == 0 && *cut: - fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line\n", l.branch()) - return 0 - case len(missing) == 0 && len(unmerged) == 0: - fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) - return 0 - } if len(missing) > 0 { fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where) - for _, v := range missing { - fmt.Fprintf(a.stdout, " %s %s\n", v.o.url(), terminalSafe(v.o.title)) + for _, text := range missing { + fmt.Fprintf(a.stdout, " %s\n", text) } } - for _, v := range unmerged { - fmt.Fprintf(a.stdout, "%s: #%d is only on open batch PR #%d, merge it before the cut: %s %s\n", l.branch(), v.o.number, b.pr.Number, v.o.url(), terminalSafe(v.o.title)) - } - for _, v := range waiting { - fmt.Fprintf(a.stdout, "%s: #%d changes %s and waits on batch PR #%d for a maintainer, bring it over or defer it before the cut: %s %s\n", l.branch(), v.o.number, workflowsDir, b.pr.Number, v.o.url(), terminalSafe(v.o.title)) + for _, text := range failing { + fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text) } return 1 } +// warnLeftBranch names the branch a closed batch's close left behind. +func (a *app) warnLeftBranch(b *batch) error { + if b.src.open || b.src.rec == nil || b.src.rec.Closing == nil { + return nil + } + c := b.src.rec.Closing + heads, err := remoteBatchHeads(b.line) + if err != nil { + return err + } + switch h, ok := heads[c.Branch]; { + case !ok: + case takenBranch(b, *c) != nil: + a.warn("%v", takenBranch(b, *c)) + case h == c.Head: + a.warn("batch pull request #%d is closed, but its branch %s is still at %.12s; backport-batch run --line %s --push deletes it", c.PR, c.Branch, h, b.line.branch()) + default: + a.warn("%v", movedBranch(*c, h)) + } + return nil +} + +// outcome is what check makes of one original. +type outcome int + +const ( + // outcomeMissing fails check and the cut: nothing accounts for an + // original the line lacks. + outcomeMissing outcome = iota + 1 + // outcomeUnmerged and outcomeWaiting fail the cut only: the cut would not + // carry what is only on the open batch, nor what waits there. + outcomeUnmerged + outcomeWaiting + outcomeWarning + outcomeNote +) + +type disposition struct { + kind outcome + text string +} + +// dispositions says what check reports about each original that needs a line +// of its own; the others pass in silence. The text is what follows the line's +// name, or the original alone for one that is missing. Originals are taken in +// merge order, each by the first of: a unit on the batch that does only part +// of it, a reason a person deferred it for, a park, a wait for a maintainer +// or on other originals, and what the record says of it. +func (b *batch) dispositions(mode analysisMode, cut bool) (map[*verdict]disposition, error) { + if c := b.src.closingOpen(); c != nil { + return nil, fmt.Errorf("batch pull request #%d is being closed; backport-batch run --line %s --push finishes it", c.PR, b.line.branch()) + } + var stale []*verdict + if b.pr != nil { + var err error + if stale, err = keptOut(b, b.picks); err != nil { + return nil, err + } + } + // A record a closed batch left still says what a maintainer decided, but + // nothing waits on a batch that is gone. + deferred, on := b.src.rec.deferred(), "" + if c := b.src.carrier; c != nil { + on = fmt.Sprintf("#%d", c.Number) + if !b.src.open { + on += " (" + prState(c) + ")" + } + } + if err := b.analyze(); err != nil { + return nil, err + } + if b.pr != nil { + if err := b.keepRecordedPartials(b.prHead); err != nil { + return nil, err + } + } + held := map[int]string{} + for n, e := range deferred { + if e.Reason != "" { + held[n] = e.Reason + } + } + states, err := b.classify(mode, held, deferred, b.src.open) + if err != nil { + return nil, err + } + // waitsOn names what an original waits on, as check says it. + waitsOn := func(s *state) string { + var names []string + for _, f := range s.needs { + names = append(names, fmt.Sprintf("#%d", f.o.number)) + } + return strings.Join(names, " and ") + } + out := map[*verdict]disposition{} + for _, v := range b.verdicts { + named := v.o.url() + " " + terminalSafe(v.o.title) + e, isDeferred := deferred[v.o.number] + s := states[v] + if s == nil { + s = &state{kind: stOwn} + } + switch { + case b.partialUnits[v] != nil && cut: + out[v] = disposition{outcomeUnmerged, fmt.Sprintf("#%d is on batch #%d, but only part of it applies to %s: it leaves %s without effect; take it out with backport-batch run --defer before the cut: %s", v.o.number, b.pr.Number, b.line.branch(), quotedFiles(b.partialUnits[v]), named)} + case b.partialUnits[v] != nil: + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d is on batch #%d, but only part of it applies to %s: it leaves %s without effect; take it out with backport-batch run --defer: %s", v.o.number, b.pr.Number, b.line.branch(), quotedFiles(b.partialUnits[v]), named)} + case slices.Contains(stale, v): + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d is still on batch #%d; take it out with backport-batch run --defer: %s", v.o.number, b.pr.Number, named)} + case v.rule == ruleDropped: + d := b.drops[v.o.number] + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d is dropped from %s on #%d (%s): %s", v.o.number, b.line.branch(), d.On, terminalSafe(d.Reason), named)} + case v.rule != 0: + case v.onBatch != "": + if cut { + out[v] = disposition{outcomeUnmerged, fmt.Sprintf("#%d is only on open batch PR #%d, merge it before the cut: %s", v.o.number, b.pr.Number, named)} + } + case s.kind == stHeld: + out[v] = disposition{outcomeWarning, fmt.Sprintf("deferred on %s to the next batch: %s (%s)", on, named, terminalSafe(e.Reason))} + case s.kind == stPartial: + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d is parked: only part of it applies to %s, which would leave %s without effect; port what applies as a pull request of its own: %s", v.o.number, b.line.branch(), quotedFiles(v.park.Files), named)} + case s.kind == stWorkflowPrimary && s.class == edgeMissing: + out[v] = disposition{outcomeMissing, named} + case s.kind == stParked: + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d is parked: it builds on %s, which never reaches %s; port what applies as a pull request of its own, or drop it: %s", v.o.number, parkNames(*v.park), b.line.branch(), named)} + case s.kind == stNeeds && s.wait == edgeMissing: + out[v] = disposition{outcomeMissing, fmt.Sprintf("%s (it waits on %s, which is not on the line either)", named, waitsOn(s))} + case s.kind == stNeeds && s.wait == edgeWorkflow && cut: + out[v] = disposition{outcomeWaiting, fmt.Sprintf("#%d waits on %s, which changes %s and waits on batch PR %s for a maintainer; bring it over, or defer both, before the cut: %s", v.o.number, waitsOn(s), workflowsDir, on, named)} + case s.kind == stNeeds && s.wait == edgeWorkflow: + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d waits on %s, which waits on %s for a maintainer: %s", v.o.number, waitsOn(s), on, named)} + case s.kind == stNeeds && s.wait == edgeDeferred: + what := "a deferred conflict on " + on + if f := s.needs[0]; deferred[f.o.number].Reason != "" { + what = "deferred on " + on + " to the next batch" + } else if states[f] != nil && states[f].kind == stNeeds { + what = "which waits itself" + } + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d waits on %s, %s: %s", v.o.number, waitsOn(s), what, named)} + case s.kind == stNeeds: + newer := "the newer line" + if b.newer != nil { + newer = b.newer.branch() + } + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d waits on %s, which is not on %s yet: %s", v.o.number, waitsOn(s), newer, named)} + case isDeferred && e.Workflows && !b.src.open: + out[v] = disposition{outcomeMissing, named} + case isDeferred && e.Workflows && cut: + out[v] = disposition{outcomeWaiting, fmt.Sprintf("#%d changes %s and waits on batch PR %s for a maintainer, bring it over or defer it before the cut: %s", v.o.number, workflowsDir, on, named)} + case isDeferred && e.Workflows: + out[v] = disposition{outcomeWarning, fmt.Sprintf("waiting on %s for a maintainer, since it changes %s: %s", on, workflowsDir, named)} + case isDeferred && len(e.Needs) == 0: + out[v] = disposition{outcomeWarning, fmt.Sprintf("deferred conflict on %s: %s", on, named)} + default: + out[v] = disposition{outcomeMissing, named} + } + } + return out, nil +} + +// parkNames names what a park waits on for a terminal. +func parkNames(p parkedEntry) string { + var out []string + for _, n := range p.Needs { + out = append(out, fmt.Sprintf("#%d", n)) + } + for _, c := range p.Commits { + out = append(out, c[:12]) + } + return strings.Join(out, ", ") +} + +// quotedFiles names files for a terminal, each quoted, the first few of them. +func quotedFiles(files []string) string { + var out []string + for _, f := range files[:min(len(files), filesListed)] { + out = append(out, strconv.Quote(f)) + } + if more := len(files) - len(out); more > 0 { + out = append(out, fmt.Sprintf("%d more", more)) + } + return strings.Join(out, ", ") +} + // terminalSafe keeps a title from carrying escape sequences into a terminal. func terminalSafe(s string) string { return capped(oneLine(clean(s, false)), titleLimit) } @@ -130,6 +315,12 @@ func (a *app) fold(args []string) int { if err != nil { return a.fail(err) } + if c := b.src.closingOpen(); c != nil { + return a.fail(fmt.Errorf("batch pull request #%d is being closed; backport-batch run --line %s --push finishes it", c.PR, l.branch())) + } + if err := a.warnLeftBranch(b); err != nil { + return a.fail(err) + } why, err := refuseFold(b, *number, *commit) if err != nil { return a.fail(err) @@ -156,7 +347,7 @@ func refuseFold(b *batch, n int, commit string) (string, error) { case v.rule == ruleSkipLabel: return fmt.Sprintf("#%d carries %s", n, labelSkip), nil case v.rule != 0: - return fmt.Sprintf("#%d is not one the batch takes: rule %d, %s", n, v.rule, v.reason), nil + return fmt.Sprintf("#%d is not one the batch takes: %s, %s", n, ruleName(v.rule), terminalSafe(v.reason)), nil case b.pr == nil: return fmt.Sprintf("no batch pull request is open on %s", b.line.branch()), nil case v.onBatch != "": @@ -192,5 +383,9 @@ func refuseFold(b *batch, n int, commit string) (string, error) { case carried != v: return fmt.Sprintf("%.12s carries #%d, not #%d", commit, carried.o.number, n), nil } - return "", nil + omitted, _, err := b.omittedEffects(v, u.parents[0], u.commit) + if err != nil || len(omitted) == 0 { + return "", err + } + return leftOut(v, b.line, omitted), nil } diff --git a/cmd/backport-batch/check_test.go b/cmd/backport-batch/check_test.go index 36cf253e8..cf66d865e 100644 --- a/cmd/backport-batch/check_test.go +++ b/cmd/backport-batch/check_test.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "regexp" "strings" "testing" "unicode" @@ -140,12 +141,22 @@ func TestCheck(t *testing.T) { { name: "a batch opened by a member with read access claims nothing", arrange: func(s *state) { - batch(s, "body\n\n"+deferred6+"\n", s.mc5) + batch(s, "body\n\n", s.mc5) s.gh.perms = map[string]string{"maintainer": "read"} }, code: 1, listed: []string{"/pull/5 ", "/pull/6 "}, }, + { + // An older record is never taken in place of a newer one nobody + // can vouch for. + name: "a record by a member with read access stops it", + arrange: func(s *state) { + batch(s, "body\n\n"+deferred6+"\n", s.mc5) + s.gh.perms = map[string]string{"maintainer": "read"} + }, + code: 2, + }, { name: "a deferred list for another line", arrange: func(s *state) { batch(s, strings.Replace(deferred6, "release-1.1", "release-1.0", 1), s.mc5) }, @@ -186,6 +197,219 @@ func TestCheck(t *testing.T) { } } +// Every category an original can be in, through the commands that put it +// there, with what check and check --cut make of it: the exit code, and the +// warning or note on stderr that leaves it 0. +func TestCheckDispositions(t *testing.T) { + plain := func(t *testing.T) *fixture { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f + } + // conflicting has #5 conflict on the line, beside #6, which picks. + conflicting := func(t *testing.T) *fixture { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f + } + must := func(t *testing.T, f *fixture, gh *fakeGH, args ...string) { + t.Helper() + if code, _, stderr := f.exec(gh, args...); code != 0 { + t.Fatalf("%q = %d\n%s", args, code, stderr) + } + f.git("switch", "-q", "main") + if len(gh.bodies) > 0 { + reopen(f, gh) + } + } + sixOnLine := func(f *fixture) { pickOnLine(f, 6, map[string]string{"app/six.txt": "6\n"}) } + cases := []struct { + name string + arrange func(t *testing.T) (*fixture, *fakeGH) + line string + check int + cut int + checkErr string // on stderr from check + cutOut string // on stdout from check --cut + cutErr string // on stderr from check --cut + notCutOut string + }{ + { + name: "missing", + arrange: func(t *testing.T) (*fixture, *fakeGH) { return plain(t), &fakeGH{} }, + check: 1, cut: 1, cutOut: "/pull/5 ", + }, + { + name: "carried", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f := plain(t) + pickOnLine(f, 5, map[string]string{"app/five.txt": "5\n"}) + sixOnLine(f) + return f, &fakeGH{} + }, + check: 0, cut: 0, notCutOut: "/pull/5", + }, + { + name: "picked, on the open batch", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := plain(t), &fakeGH{} + openBatch(t, f, gh) + return f, gh + }, + check: 0, cut: 1, cutOut: "#5 is only on open batch PR #90", + }, + { + name: "deferred conflict, open record", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := conflicting(t), &fakeGH{} + openBatch(t, f, gh) + sixOnLine(f) + return f, gh + }, + check: 0, cut: 0, checkErr: "warning: deferred conflict on #90: https://github.com/cozystack/cozystack/pull/5 ", cutErr: "warning: deferred conflict on #90", + }, + { + name: "deferred with a reason, open record", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := plain(t), &fakeGH{} + openBatch(t, f, gh) + must(t, f, gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the app e2e", "--push") + sixOnLine(f) + return f, gh + }, + check: 0, cut: 0, checkErr: "warning: deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ", cutErr: "(fails the app e2e)", + }, + { + name: "workflow wait, open record, beside a pick", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := workflowWorld(t), &fakeGH{} + must(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + return f, gh + }, + check: 0, cut: 1, checkErr: "warning: waiting on #90 for a maintainer", cutOut: "#5 changes .github/workflows/ and waits on batch PR #90", + }, + { + name: "workflow wait, open record, alone", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := workflowWorld(t), &fakeGH{} + must(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + pickOnLine(f, 6, map[string]string{"app/six.txt": "6\n", ".github/renovate.json": "{}\n"}) + return f, gh + }, + check: 0, cut: 1, checkErr: "warning: waiting on #90 for a maintainer", cutOut: "#5 changes .github/workflows/ and waits on batch PR #90", + }, + { + name: "backport/skip, not picked", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f := plain(t) + sixOnLine(f) + return f, &fakeGH{labels: map[string][]int{labelSkip: {5}}} + }, + check: 0, cut: 0, notCutOut: "/pull/5", + }, + { + name: "backport/skip after the pick, still on the open batch", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f, gh := plain(t), &fakeGH{} + openBatch(t, f, gh) + sixOnLine(f) + gh.labels = map[string][]int{labelSkip: {5}} + return f, gh + }, + check: 0, cut: 0, checkErr: "warning: #5 is still on batch #90; take it out with backport-batch run --defer", cutErr: "still on batch #90", + }, + { + name: "rule 6", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + return f, &fakeGH{} + }, + line: "release-1.0", + check: 0, cut: 0, notCutOut: "/pull/5", + }, + { + name: "rules 2 to 4", + arrange: func(t *testing.T) (*fixture, *fakeGH) { + f := newFixture(t) + f.lines() + f.pr(5, "feat(app): five", "", map[string]string{"app/five.txt": "5\n"}) + return f, &fakeGH{} + }, + check: 0, cut: 0, notCutOut: "/pull/5", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, gh := tc.arrange(t) + line := tc.line + if line == "" { + line = "release-1.1" + } + code, stdout, stderr := f.exec(gh, "check", "--line", line) + if code != tc.check || !strings.Contains(stderr, tc.checkErr) { + t.Errorf("check = %d, want %d with %q\nstdout:\n%s\nstderr:\n%s", code, tc.check, tc.checkErr, stdout, stderr) + } + code, stdout, stderr = f.exec(gh, "check", "--line", line, "--cut") + if code != tc.cut || !strings.Contains(stdout, tc.cutOut) || !strings.Contains(stderr, tc.cutErr) { + t.Errorf("check --cut = %d, want %d with %q, %q\nstdout:\n%s\nstderr:\n%s", code, tc.cut, tc.cutOut, tc.cutErr, stdout, stderr) + } + if tc.notCutOut != "" && strings.Contains(stdout, tc.notCutOut) { + t.Errorf("check --cut lists %q:\n%s", tc.notCutOut, stdout) + } + if tc.cut == 0 && !strings.Contains(stdout, "no change the policy takes is missing before the cut") { + t.Errorf("check --cut passes without saying so:\n%s", stdout) + } + }) + } +} + +// Output names a rule as the policy does: the numbers the tool keeps for +// what is no rule of the policy never reach a reader. +func TestOutputNamesNoInternalRule(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.setPolicy("release-1.1: 2099-01-01\n") + gh := &fakeGH{labels: map[string][]int{labelInclude: {6}}} + head := openBatch(t, f, gh) + internal := regexp.MustCompile(`rule 1[0-9][0-9]`) + for _, args := range [][]string{ + {"fold", "--line", "release-1.1", "--original", "5", "--commit", head}, + {"run", "--line", "release-1.1", "--defer", "5", "--reason", "x"}, + } { + code, stdout, stderr := f.exec(gh, args...) + f.git("switch", "-q", "main") + if code == 0 || !strings.Contains(stdout+stderr, "#5 is not one the batch takes: rule 1, before the line's start, merged ") { + t.Errorf("%s = %d, want a refusal naming rule 1:\n%s%s", args[0], code, stdout, stderr) + } + if m := internal.FindString(stdout + stderr); m != "" { + t.Errorf("%s prints %q:\n%s%s", args[0], m, stdout, stderr) + } + } +} + +// A refusal fold prints is one line with no control sequence, whatever the +// reason it repeats says: a drop's reason comes from a pull request body, a +// rule 4 reason from file names an original's author chose. +func TestFoldRefusalPrintsNothingThatDrivesATerminal(t *testing.T) { + l, _ := parseLine("release-1.1") + for _, reason := range []string{"dropped on #90: x\n\x1b[2Jrefused: forged", "touches \"app/a\x1b]0;x\x07\nok\""} { + b := &batch{line: l, verdicts: []*verdict{{o: &original{number: 5}, rule: ruleDropped, reason: reason}}} + why, err := refuseFold(b, 5, strings.Repeat("a", 40)) + if err != nil || why == "" || strings.ContainsFunc(why, func(r rune) bool { return unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) { + t.Errorf("refuseFold = %q, %v; want one clean line", why, err) + } + } +} + type errString string func (e errString) Error() string { return string(e) } diff --git a/cmd/backport-batch/cut_test.go b/cmd/backport-batch/cut_test.go new file mode 100644 index 000000000..02466b304 --- /dev/null +++ b/cmd/backport-batch/cut_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + "testing" +) + +// pickOnLine lands original n on release-1.1 as a maintainer's pick of its +// merge commit, with the -x line git appends, so the line carries it. +func pickOnLine(f *fixture, n int, files map[string]string) { + f.t.Helper() + mc, _, _ := strings.Cut(f.git("log", "--first-parent", "--format=%H", "--grep", fmt.Sprintf("(#%d)$", n), "main"), "\n") + f.onLine("release-1.1", fmt.Sprintf("fix: original %d\n\n(cherry picked from commit %s)", n, mc), files) +} + +// A workflow original that waits on the open batch fails the cut, also when +// it is all the batch has left: the pick beside it was labelled +// backport/skip after the pick, or reached the line by hand. +func TestCutFailsWhenOnlyAWorkflowWaitIsLeft(t *testing.T) { + for _, tc := range []struct { + name string + leave func(f *fixture, gh *fakeGH) + }{ + {"skip after the pick", func(_ *fixture, gh *fakeGH) { gh.labels = map[string][]int{labelSkip: {6}} }}, + {"picked onto the line by hand", func(f *fixture, _ *fakeGH) { + pickOnLine(f, 6, map[string]string{"app/six.txt": "6\n", ".github/renovate.json": "{}\n"}) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := workflowWorld(t) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + tc.leave(f, gh) + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90 for a maintainer") { + t.Errorf("check --cut = %d, want 1 naming #5:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + if strings.Contains(stdout, "on the line or on the batch") { + t.Errorf("check --cut passes the line while #5 waits:\n%s", stdout) + } + }) + } +} + +// A conflict the batch deferred is still deferred once that batch merged: +// the next run picks nothing, opens nothing and writes nothing to the merged +// pull request, and check reads the merged batch's record. +func TestCheckAfterTheBatchMerged(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mergeBatch(f, gh) + written := len(gh.created) + len(gh.edited) + len(gh.issueComments) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 { + t.Fatalf("run --continue --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if got := len(gh.created) + len(gh.edited) + len(gh.issueComments); got != written || f.remoteHead(batchBranch1) != "" { + t.Errorf("the run picked nothing and still wrote: created %q, edited %q", gh.created, gh.edited) + } + for _, cut := range []bool{false, true} { + args := []string{"check", "--line", "release-1.1"} + if cut { + args = append(args, "--cut") + } + code, stdout, stderr := f.exec(gh, args...) + if code != 0 || !strings.Contains(stderr, "warning: deferred conflict on #90 (merged): https://github.com/cozystack/cozystack/pull/5 ") { + t.Errorf("%q = %d, want 0 with #5 a deferred conflict:\nstdout:\n%s\nstderr:\n%s", args, code, stdout, stderr) + } + } +} + +// A workflow original waits only while a batch is open to list it: once the +// batch that listed it merged, it is missing. +func TestWorkflowWaitAfterTheBatchMergedIsMissing(t *testing.T) { + f := workflowWorld(t) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + mergeBatch(f, gh) + for _, cut := range []bool{false, true} { + args := []string{"check", "--line", "release-1.1"} + if cut { + args = append(args, "--cut") + } + if code, stdout, stderr := f.exec(gh, args...); code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("%q = %d, want 1 with #5 missing:\nstdout:\n%s\nstderr:\n%s", args, code, stdout, stderr) + } + } +} diff --git a/cmd/backport-batch/defer.go b/cmd/backport-batch/defer.go index ab9741b00..de77d5d5c 100644 --- a/cmd/backport-batch/defer.go +++ b/cmd/backport-batch/defer.go @@ -27,23 +27,29 @@ import ( // parseDefer reads --defer N[,N] and the --reason that has to come with it. func parseDefer(list, reason string) ([]int, string, error) { if list == "" { - return nil, "", errors.New("--reason goes with --defer") + return nil, "", errors.New("--reason goes with --defer or --drop") } reason = capped(oneLine(clean(reason, false)), titleLimit) if reason == "" { return nil, "", errors.New("--defer needs a --reason, which the batch body records") } + numbers, err := parseNumbers("--defer", list) + return numbers, reason, err +} + +// parseNumbers reads the N[,N] list a flag takes, each once. +func parseNumbers(flag, list string) ([]int, error) { var numbers []int for _, s := range strings.Split(list, ",") { n, err := strconv.Atoi(strings.TrimPrefix(strings.TrimSpace(s), "#")) if err != nil || n <= 0 { - return nil, "", fmt.Errorf("--defer %q is not a list of pull request numbers", list) + return nil, fmt.Errorf("%s %q is not a list of pull request numbers", flag, list) } if !slices.Contains(numbers, n) { numbers = append(numbers, n) } } - return numbers, reason, nil + return numbers, nil } // rebuild takes the originals in numbers out of the open batch: it builds the @@ -54,17 +60,8 @@ func (a *app) rebuild(b *batch, numbers []int, reason string, push bool) int { if b.pr == nil { return a.fail(fmt.Errorf("no batch pull request is open on %s; --defer rebuilds the open one", b.line.branch())) } - branch := b.pr.Head.Ref - date, err := branchDate(branch) - if err != nil { - return a.fail(err) - } - recorded, err := parseDeferred(b.pr.Body, b.line) - if err != nil { - return a.fail(fmt.Errorf("batch pull request #%d: %w", b.pr.Number, err)) - } hold := map[int]string{} - for n, e := range recorded { + for n, e := range b.recorded() { if e.Reason != "" { hold[n] = e.Reason } @@ -81,12 +78,20 @@ func (a *app) rebuild(b *batch, numbers []int, reason string, push bool) int { switch v := b.find(n); { case v == nil: return a.fail(fmt.Errorf("#%d did not merge into main after %s branched", n, b.line.branch())) - case v.rule != 0 && !slices.Contains(stale, v): - return a.fail(fmt.Errorf("#%d is not one the batch takes: rule %d, %s", n, v.rule, terminalSafe(v.reason))) + case v.rule != 0 && !slices.Contains(stale, v) && b.partialUnits[v] == nil: + return a.fail(fmt.Errorf("#%d is not one the batch takes: %s, %s", n, ruleName(v.rule), terminalSafe(v.reason))) } hold[n] = reason } - start, replay, err := rebuildPlan(b, hold) + return a.reassemble(b, hold, hold, b.src.rec.dropped(), modeDefer, push) +} + +// reassemble builds the open batch again off the line's head without the +// units of the originals in out, and reports it with hold deferred for their +// reasons and drops kept. +func (a *app) reassemble(b *batch, out, hold map[int]string, drops []droppedEntry, mode string, push bool) int { + branch := b.pr.Head.Ref + start, replay, err := rebuildPlan(b, out) if err != nil { return a.fail(err) } @@ -102,7 +107,7 @@ func (a *app) rebuild(b *batch, numbers []int, reason string, push bool) int { case err != nil: return a.fail(err) case failed != "": - return a.fail(fmt.Errorf("without what is deferred, %.12s%s no longer applies (%s); defer it as well", failed, originName(b, u), strings.Join(files, ", "))) + return a.fail(fmt.Errorf("without what is deferred, %.12s%s no longer applies (%s); defer it as well", failed, originName(b, u), terminalSafe(strings.Join(files, ", ")))) } } head, err := resolve("HEAD") @@ -113,24 +118,90 @@ func (a *app) rebuild(b *batch, numbers []int, reason string, push bool) int { if err != nil { return a.fail(err) } + return a.reclassify(b, picks, hold, drops, mode, push) +} + +// reclassify reports the open batch as picks shows it, after an operation +// that adds no original: what the record held stays held, and an original +// the batch neither carries nor records is missing until a run tries it. +func (a *app) reclassify(b *batch, picks *history, hold map[int]string, drops []droppedEntry, mode string, push bool) int { + date, err := branchDate(b.pr.Head.Ref) + if err != nil { + return a.fail(err) + } + recorded := b.recorded() r := newReport(b) + // None of these changes how the batch is assembled, so the record keeps + // the flag the batch was assembled with. + r.opts = runOptions{NoWorkflowFiles: b.src.savedNoWorkflowFiles(), Mode: mode} + r.dropped = drops if r.stale, err = keptOut(b, picks); err != nil { return a.fail(err) } + // What the rebuild took off is no longer there for what builds on it. + units, err := b.unitsOn(picks.head) + if err != nil { + return a.fail(err) + } + for _, v := range b.verdicts { + v.batched = units[v] + } + on := map[*verdict]bool{} + for _, v := range b.qualifying() { + if on[v], err = b.onBranch(picks, v); err != nil { + return a.fail(err) + } + if !on[v] { + v.onBatch = "" + } + } + if err := b.keepRecordedPartials(picks.head); err != nil { + return a.fail(err) + } + states, err := b.classify(analysisMode{noWorkflowFiles: r.opts.NoWorkflowFiles}, hold, recorded, true) + if err != nil { + return a.fail(err) + } for _, v := range b.qualifying() { e := recorded[v.o.number] + s := states[v] + if s == nil { + s = &state{kind: stOwn} + } switch { - case found(picks, v): + case on[v]: r.onBranch = append(r.onBranch, v) case hold[v.o.number] != "": r.held = append(r.held, pick{v: v, status: "held", reason: hold[v.o.number]}) + case s.kind == stPartial || s.kind == stParked: + r.parked = append(r.parked, v) + case s.kind == stNeeds: + r.needs = append(r.needs, pick{v: v, status: "waiting", needs: numbers(s.needs)}) + case s.kind == stWorkflowPrimary && !e.Workflows: + r.missing = append(r.missing, v) case e.Workflows: r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: e.Files, more: e.More}) - case len(e.Files) > 0: + case len(e.Files) > 0 && len(e.Needs) == 0: r.deferred = append(r.deferred, pick{v: v, status: "conflict", files: e.Files, more: e.More, failed: e.Commit}) + default: + // Merged into main since the batch was assembled, or recorded + // nowhere: a rebuild adds no original, the next run tries it. + r.missing = append(r.missing, v) } } - return a.finish(b, r, branch, date, push, true) + if err := b.markPartial(r, picks.head); err != nil { + return a.fail(err) + } + return a.finish(b, r, b.pr.Head.Ref, date, push, true) +} + +// recorded is what the open batch's record defers; a closed batch's record +// holds nothing back on this one. +func (b *batch) recorded() map[int]deferredEntry { + if !b.src.open { + return map[int]deferredEntry{} + } + return b.src.rec.deferred() } // rebuildPlan is where the rebuild of the open batch without the originals in diff --git a/cmd/backport-batch/defer_test.go b/cmd/backport-batch/defer_test.go index 856c9909a..66f7807a0 100644 --- a/cmd/backport-batch/defer_test.go +++ b/cmd/backport-batch/defer_test.go @@ -261,8 +261,8 @@ func TestDeferLeasesTheHeadItRebuilt(t *testing.T) { pr := f.batchPR(90, "release-1.1", batchBranch1, gh.prs[0].Body) a := &app{gh: gh} - b := &batch{line: l, lineHead: lineHead, pr: &pr, prHead: head} - err := a.publish(b, batchBranch1, rebuilt, "title", "body", true) + b := &batch{line: l, lineHead: lineHead, pr: &pr, prHead: head, src: recordSource{carrier: &pr, open: true, code: &pr}} + err := a.publish(b, publication{branch: batchBranch1, head: rebuilt, title: "title", body: "body"}, true) if err == nil || !strings.Contains(err.Error(), "which the rebuild started from") { t.Errorf("publish over a head pushed since = %v, want a refusal", err) } @@ -378,3 +378,332 @@ func TestSkipAfterThePick(t *testing.T) { t.Errorf("check = %d\n%s", code, stderr) } } + +// closeWorld is a batch carrying #5 alone, opened and pushed. +func closeWorld(t *testing.T) (*fixture, *fakeGH, string) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + h1 := openBatch(t, f, gh) + return f, gh, h1 +} + +// Deferring everything a batch carries closes it: the record says why and +// what it froze while the pull request is still open, then the pull request +// is closed and its branch deleted, leased on that head. The next batch +// tries what was deferred again. +func TestDeferEverythingClosesTheBatch(t *testing.T) { + f, gh, h1 := closeWorld(t) + var log []string + gh.after = map[string]func(){ + "createComment": func() { log = append(log, "comment") }, + "editPull": func() { log = append(log, "edit") }, + "closePull": func() { log = append(log, "close, branch at "+f.remoteHead(batchBranch1)) }, + } + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the app e2e", "--push") + if code != 0 || !strings.Contains(stderr, "closed https://github.com/cozystack/cozystack/pull/90: without what is deferred it carries nothing beyond release-1.1; the next batch tries them again") { + t.Fatalf("run --defer of the last pick = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if want := []string{"comment", "edit", "close, branch at " + h1}; !slices.Equal(log, want) || f.remoteHead(batchBranch1) != "" || gh.find(90).State != "closed" { + t.Errorf("writes %q, branch at %q, #90 %s; want %q, the branch deleted and #90 closed", log, f.remoteHead(batchBranch1), gh.find(90).State, want) + } + rec, err := parseRecord(gh.find(90).Body, notifyLine) + if err != nil || rec.Closing == nil || *rec.Closing != (closing{PR: 90, Branch: batchBranch1, Head: h1}) || rec.Built.Options.Mode != modeClose || rec.deferred()[5].Reason != "fails the app e2e" { + t.Errorf("the closed batch's record is %+v, %v; want #5 deferred and the close it froze", rec, err) + } + if !strings.Contains(stdout, `"closing":{"pr":90,"branch":"`+batchBranch1+`","head":"`+h1+`"}`) { + t.Errorf("the body printed does not say what the close froze:\n%s", stdout) + } + code, _, stderr = f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "warning: deferred on #90 (closed) to the next batch: https://github.com/cozystack/cozystack/pull/5 ") { + t.Errorf("check = %d, want 0 with #5 deferred:\n%s", code, stderr) + } + gh.nextPR = 91 + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): five (#5)"}) || gh.find(91) == nil { + t.Errorf("the next batch carries %q, want #5 tried again on a new pull request", got) + } +} + +// A close that stopped part way is finished by run --push, and by nothing +// else, and never past a head it did not freeze; nothing is ever written to +// the closed pull request. +func TestCloseRecovery(t *testing.T) { + deferAll := []string{"run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the app e2e", "--push"} + push := []string{"run", "--line", "release-1.1", "--continue", "--push"} + // moveBranch pushes a commit onto the batch branch, or recreates it, as + // someone else would. + moveBranch := func(f *fixture, base string) string { + f.git("switch", "-q", "--detach", base) + f.git("commit", "-q", "--allow-empty", "-m", "pushed since") + h2 := f.git("rev-parse", "HEAD") + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + return h2 + } + writes := func(gh *fakeGH) int { return len(gh.issueComments) + len(gh.edited) } + + t.Run("stopped after the record, then the branch moved", func(t *testing.T) { + f, gh, h1 := closeWorld(t) + gh.after = map[string]func(){"editPull": func() { gh.fail = map[string]error{"pull": errString("HTTP 502")} }} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 || !strings.Contains(stderr, "HTTP 502") { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + h2 := moveBranch(f, h1) + before := writes(gh) + if code, _, stderr := f.exec(gh, push...); code != 2 || !strings.Contains(stderr, h2[:12]) { + t.Errorf("run --push over a moved branch = %d, want 2 naming it:\n%s", code, stderr) + } + if gh.find(90).State != "open" || f.remoteHead(batchBranch1) != h2 || writes(gh) != before { + t.Error("the run acted on a branch that moved since the close froze it") + } + }) + // GitHub and the remote are read apart, and either may be the first to + // show that someone pushed while the record was written. + for _, seen := range []string{"by the remote", "by GitHub"} { + t.Run("the branch moved while the record was written, seen "+seen, func(t *testing.T) { + f, gh, h1 := closeWorld(t) + gh.after = map[string]func(){"editPull": func() { + gh.remoteHead = func(string) string { return "" } + if seen == "by GitHub" { + gh.find(90).Head.SHA = strings.Repeat("b", 40) + } else { + moveBranch(f, h1) + } + }} + code, _, stderr := f.exec(gh, deferAll...) + f.git("switch", "-q", "main") + if code != 2 || !strings.Contains(stderr, "which its record froze") || gh.find(90).State != "open" || f.remoteHead(batchBranch1) == "" { + t.Errorf("run = %d, #90 %s, branch %q; want 2, #90 open and the branch kept:\n%s", code, gh.find(90).State, f.remoteHead(batchBranch1), stderr) + } + }) + } + t.Run("the close failed", func(t *testing.T) { + f, gh, _ := closeWorld(t) + gh.fail = map[string]error{"closePull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + before := writes(gh) + // Anything but run --push leaves it as it is. + for _, args := range [][]string{ + {"run", "--line", "release-1.1", "--continue"}, + {"check", "--line", "release-1.1"}, + {"fold", "--line", "release-1.1", "--original", "5", "--commit", strings.Repeat("a", 40)}, + {"notify", "--line", "release-1.1", "--push"}, + } { + if code, _, stderr := f.exec(gh, args...); code != 2 || !strings.Contains(stderr, "being closed") { + t.Errorf("%q = %d, want 2:\n%s", args, code, stderr) + } + f.git("switch", "-q", "main") + } + if writes(gh) != before || gh.find(90).State != "open" { + t.Error("a command other than run --push wrote to the batch") + } + code, _, stderr := f.exec(gh, push...) + if code != 0 || gh.find(90).State != "closed" || f.remoteHead(batchBranch1) != "" || writes(gh) != before { + t.Errorf("run --push = %d, #90 %s, branch %q, %d writes; want the close finished and nothing else\n%s", code, gh.find(90).State, f.remoteHead(batchBranch1), writes(gh)-before, stderr) + } + }) + // A run that finishes a close stops there: the change it was asked for + // needs an open batch pull request, and none is left to make it on. + t.Run("a change asked for while the close was left", func(t *testing.T) { + f, gh, _ := closeWorld(t) + gh.fail = map[string]error{"closePull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + code, _, stderr := f.exec(gh, deferAll...) + f.git("switch", "-q", "main") + if code != 2 || !strings.Contains(stderr, "finished the close of #90 and made no other change") || gh.find(90).State != "closed" || f.remoteHead(batchBranch1) != "" { + t.Errorf("run --defer = %d, #90 %s, branch %q; want 2, the close finished and nothing deferred:\n%s", code, gh.find(90).State, f.remoteHead(batchBranch1), stderr) + } + }) + t.Run("closed by hand while the close writes", func(t *testing.T) { + f, gh, h1 := closeWorld(t) + gh.after = map[string]func(){"createComment": func() { gh.find(90).State = "closed" }} + code, _, stderr := f.exec(gh, deferAll...) + f.git("switch", "-q", "main") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || len(gh.edited) != 0 || f.remoteHead(batchBranch1) != h1 { + t.Errorf("run = %d, edited %q, branch %q; want 2, no edit and the branch kept:\n%s", code, gh.edited, f.remoteHead(batchBranch1), stderr) + } + }) + t.Run("the delete failed", func(t *testing.T) { + f, gh, h1 := closeWorld(t) + gh.after = map[string]func(){"closePull": func() { f.refusePushes(true) }} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + f.refusePushes(false) + code, _, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "warning: batch pull request #90 is closed, but its branch "+batchBranch1+" is still at "+h1[:12]) { + t.Errorf("check = %d, want 0 naming the branch:\n%s", code, stderr) + } + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue"); code != 2 { + t.Errorf("run without --push = %d, want 2:\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if code, _, stderr := f.exec(gh, push...); code != 0 || f.remoteHead(batchBranch1) != "" { + t.Errorf("run --push = %d, branch %q; want it deleted\n%s", code, f.remoteHead(batchBranch1), stderr) + } + mustRun(t, f, gh, "check", "--line", "release-1.1") + }) + // A batch pull request a person opens on the branch a close left owns + // it: neither recovery nor check offers to delete it, moved or not. One + // on another branch does not stop the delete. + for _, tc := range []struct { + name string + branch string + moved bool + }{ + {"a batch opened on the branch the close left", batchBranch1, false}, + {"a batch opened on the branch the close left, moved since", batchBranch1, true}, + {"a batch opened on another branch", "backport-batch/release-1.1-20261027", false}, + } { + t.Run(tc.name, func(t *testing.T) { + f, gh, h1 := closeWorld(t) + gh.after = map[string]func(){"closePull": func() { f.refusePushes(true) }} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + f.refusePushes(false) + gh.after = nil + if tc.branch != batchBranch1 { + f.git("push", "-q", "origin", h1+":refs/heads/"+tc.branch) + } + left := h1 + if tc.moved { + left = moveBranch(f, h1) + } + gh.prs = append(gh.prs, f.batchPR(91, "release-1.1", tc.branch, "a maintainer's batch")) + taken := tc.branch == batchBranch1 + _, _, stderr := f.exec(gh, "check", "--line", "release-1.1") + f.git("switch", "-q", "main") + if named := strings.Contains(stderr, "batch pull request #91 is open on "+batchBranch1); named != taken || taken == strings.Contains(stderr, "--push deletes it") || taken && strings.Contains(stderr, "git push origin --delete") { + t.Errorf("check names the branch as #91's: %v, want %v, and offers no delete of it:\n%s", named, taken, stderr) + } + code, _, stderr := f.exec(gh, push...) + f.git("switch", "-q", "main") + if taken && (code != 2 || !strings.Contains(stderr, "batch pull request #91 is open on "+batchBranch1) || strings.Contains(stderr, "git push origin --delete") || f.remoteHead(batchBranch1) != left || gh.find(91).State != "open") { + t.Errorf("run --push = %d, branch %q; want 2, no delete offered and #91's branch kept:\n%s", code, f.remoteHead(batchBranch1), stderr) + } + if !taken && (code != 0 || f.remoteHead(batchBranch1) != "") { + t.Errorf("run --push = %d, branch %q; want the left branch deleted:\n%s", code, f.remoteHead(batchBranch1), stderr) + } + }) + } + t.Run("the branch moved after the close", func(t *testing.T) { + f, gh, h1 := closeWorld(t) + var h2 string + gh.after = map[string]func(){"closePull": func() { h2 = moveBranch(f, h1) }} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 || !strings.Contains(stderr, "is at "+h2[:12]+" now") { + t.Fatalf("run = %d, want 2 naming where the branch moved\n%s", code, stderr) + } + f.git("switch", "-q", "main") + // What was pushed onto the branch may be work the next run cannot + // pick again, so the refusal promises nothing about deleting it. + if code, _, stderr := f.exec(gh, push...); code != 2 || !strings.Contains(stderr, h1[:12]) || !strings.Contains(stderr, h2[:12]) || !strings.Contains(stderr, "#90") || strings.Contains(stderr, "nothing on it is lost") || !strings.Contains(stderr, "check what "+h2[:12]+" carries") { + t.Errorf("run --push = %d, want 2 naming both heads and #90, and what to check before deleting:\n%s", code, stderr) + } + if f.remoteHead(batchBranch1) != h2 { + t.Error("the run moved a branch it did not freeze") + } + if _, err := git(deleteArgs(batchBranch1, h1)...); err == nil || f.remoteHead(batchBranch1) != h2 { + t.Error("the remote took a delete whose lease names a head it no longer has") + } + }) + t.Run("a batch of the same day that opened no pull request", func(t *testing.T) { + f, gh, _ := closeWorld(t) + if code, _, stderr := f.exec(gh, deferAll...); code != 0 || f.remoteHead(batchBranch1) != "" { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + // The same day, a new batch reuses the branch name, pushes, and + // fails to open its pull request. + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh.fail = map[string]error{"createPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, push...); code != 2 { + t.Fatalf("the same-day run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + h2 := f.remoteHead(batchBranch1) + for _, args := range [][]string{push, {"run", "--line", "release-1.1", "--onto", h2, "--push"}} { + if code, _, stderr := f.exec(gh, args...); code != 2 || !strings.Contains(stderr, "git push origin --delete "+batchBranch1) { + t.Errorf("%q = %d, want 2 naming the branch to delete:\n%s", args, code, stderr) + } + f.git("switch", "-q", "main") + } + if f.remoteHead(batchBranch1) != h2 || len(gh.created) != 1 { + t.Errorf("the branch moved to %.12s or a pull request was opened: %q", f.remoteHead(batchBranch1), gh.created) + } + f.git("push", "-q", "origin", "--delete", batchBranch1) + gh.nextPR = 91 + mustRun(t, f, gh, push...) + if gh.find(91) == nil { + t.Error("after the branch was deleted the next run opened no batch") + } + }) + t.Run("a preview", func(t *testing.T) { + f, gh, _ := closeWorld(t) + gh.fail = map[string]error{"closePull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + before := writes(gh) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "x") + if code != 2 || !strings.Contains(stderr, "run --line release-1.1 --push finishes") || writes(gh) != before || gh.find(90).State != "open" { + t.Errorf("a preview = %d, want 2 naming the step left and nothing written:\n%s", code, stderr) + } + }) +} + +// A batch that merged on GitHub and left its branch behind is named when a +// run of the same day refuses to push over that branch. +func TestRefusalNamesTheMergedBatchsBranch(t *testing.T) { + f, gh, _ := closeWorld(t) + head := f.remoteHead(batchBranch1) + mergeBatch(f, gh) + // A commit pushed onto the batch after it merged is left on its branch. + f.git("switch", "-q", "--detach", head) + f.git("commit", "-q", "--allow-empty", "-m", "pushed after the merge") + pushed := f.git("rev-parse", "HEAD") + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 2 || !strings.Contains(stderr, "it is the branch of batch pull request #90, which is merged") || strings.Contains(stderr, "--onto") || !strings.Contains(stderr, "check what "+pushed[:12]+" carries") { + t.Errorf("run --push = %d, want a refusal naming #90:\n%s", code, stderr) + } +} + +// A rebuild that stops names the files that no longer apply on the line of +// its refusal, whatever the files are called. +func TestDeferRefusalNamesFilesOnOneLine(t *testing.T) { + f := newFixture(t) + f.lines() + name := "app/x\nwarning: forged.txt" + f.pr(5, "fix(app): five", "", map[string]string{name: "five\n"}) + f.pr(6, "fix(app): six", "", map[string]string{name: "five and six\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + f.onLine("release-1.1", "fix(chart): a line-only value", map[string]string{"chart/values.yaml": "replicas: 3\nimage: app:1\n"}) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "x", "--push") + f.git("switch", "-q", "main") + if code != 2 || !strings.Contains(stderr, "no longer applies") { + t.Fatalf("run --defer 5 = %d, want the refusal:\n%s", code, stderr) + } + for _, ln := range strings.Split(stderr, "\n") { + if strings.HasPrefix(ln, "warning: forged") { + t.Errorf("the refusal split onto a line of its own: %q", ln) + } + } +} diff --git a/cmd/backport-batch/deps.go b/cmd/backport-batch/deps.go new file mode 100644 index 000000000..895897709 --- /dev/null +++ b/cmd/backport-batch/deps.go @@ -0,0 +1,912 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// What an original builds on that its line lacks is read from git alone, in +// two ways: a file it modifies or deletes that the line does not have, named +// by the first-parent commit on main that added it (V1), and a line it +// removes or changes that git blame gives to a commit the line lacks, kept +// only while that line's text is absent from the line's file (V2c). Either +// names the first-parent commit that brought the evidence to main: the merge +// of another original, or a commit that merged no pull request. + +// Bounds per original, past which the evidence is incomplete rather than +// read at any cost. +const ( + maxDepFiles = 256 + maxBlamedLines = 20000 + maxDepBlob = 8 << 20 + gitDeadline = 2 * time.Minute +) + +// dependency is a first-parent commit on main an original builds on, with +// the original's paths the evidence is in. +type dependency struct { + merge string + // v is the original merge merged, nil for a commit that merged none. + v *verdict + paths []string + // added says some of the evidence is a file the line lacks (V1). + added bool +} + +// errDeadline is a git call that ran out of time: incomplete evidence, not +// a failure. +var errDeadline = errors.New("git took longer than " + gitDeadline.String()) + +// gitTimed runs git under the deadline the detector gives each call. +func gitTimed(args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), gitDeadline) + defer cancel() + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(cmd.Environ(), commandEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + err := cmd.Run() + switch { + case ctx.Err() != nil: + return "", errDeadline + case err != nil: + return "", fmt.Errorf("git %s: %w: %s", quotedArgs(args), err, terminalSafe(errOut.String())) + } + return out.String(), nil +} + +// quotedArgs names a git command's arguments for an error, each quoted: a +// file name can hold a newline, and the error reaches a terminal and a CI +// log, which would take the rest for a line of their own. +func quotedArgs(args []string) string { + out := make([]string, len(args)) + for i, a := range args { + out[i] = strconv.Quote(a) + } + return strings.Join(out, " ") +} + +// gitStatusTimed is gitStatus under the detector's deadline: a non-zero +// exit is an answer, and running out of time is errDeadline. +func gitStatusTimed(args ...string) (stdout, stderr string, code int, err error) { + ctx, cancel := context.WithTimeout(context.Background(), gitDeadline) + defer cancel() + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(cmd.Environ(), commandEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + err = cmd.Run() + var ee *exec.ExitError + switch { + case ctx.Err() != nil: + return "", "", -1, errDeadline + case errors.As(err, &ee): + return out.String(), errOut.String(), ee.ExitCode(), nil + case err != nil: + return "", "", -1, fmt.Errorf("git %s: %w", quotedArgs(args), err) + } + return out.String(), errOut.String(), 0, nil +} + +// broughtBy maps every commit in base..mainHead to the first-parent commit +// that brought it to main: each first-parent commit claims itself, then +// every commit of the range reachable from its other parents that no +// earlier one claimed. A commit outside the map is an ancestor of base. +func broughtBy(base, mainHead string) (map[string]string, error) { + first, err := revList("--first-parent", "--reverse", "--end-of-options", base+".."+mainHead) + if err != nil { + return nil, err + } + out, err := git("rev-list", "--parents", "--topo-order", "--end-of-options", base+".."+mainHead) + if err != nil { + return nil, err + } + parents := map[string][]string{} + for ln := range strings.Lines(out) { + f := strings.Fields(ln) + if len(f) == 0 { + continue + } + for _, sha := range f { + if checkSHA(sha) != nil { + return nil, fmt.Errorf("rev-list: %q is not a commit", sha) + } + } + parents[f[0]] = f[1:] + } + by := map[string]string{} + for _, fp := range first { + if _, ok := parents[fp]; !ok { + return nil, fmt.Errorf("rev-list: %s is on main's first-parent line but not in its range", fp) + } + by[fp] = fp + stack := slices.Clone(parents[fp][min(1, len(parents[fp])):]) + for len(stack) > 0 { + c := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if _, inRange := parents[c]; !inRange || by[c] != "" { + continue + } + by[c] = fp + stack = append(stack, parents[c]...) + } + } + return by, nil +} + +// hunk is an old range of a diff, a,+b in git blame's terms. +type hunk struct{ start, count int } + +var hunkRE = regexp.MustCompile(`^@@ -(\d+)(?:,(\d+))? \+\d+(?:,\d+)? @@`) + +// oldRanges reads the removed or changed line ranges of a -U0 diff of one +// file, and whether git called it binary. A header it cannot read is an +// error, never evidence of absence. +func oldRanges(diff string) ([]hunk, bool, error) { + var out []hunk + for ln := range strings.Lines(diff) { + ln = strings.TrimSuffix(ln, "\n") + switch { + case strings.HasPrefix(ln, "Binary files ") && strings.HasSuffix(ln, " differ"): + return nil, true, nil + case strings.HasPrefix(ln, "@@"): + m := hunkRE.FindStringSubmatch(ln) + if m == nil { + return nil, false, fmt.Errorf("unreadable hunk header %q", terminalSafe(ln)) + } + start, _ := strconv.Atoi(m[1]) + count := 1 + if m[2] != "" { + count, _ = strconv.Atoi(m[2]) + } + if count > 0 { + out = append(out, hunk{start, count}) + } + } + } + return out, false, nil +} + +// blameArgs blames ranges of file at rev, a full commit, which git takes for +// a revision whatever follows: blame refuses --end-of-options. +func blameArgs(rev, file string, ranges []hunk) ([]string, error) { + if err := checkSHA(rev); err != nil { + return nil, err + } + args := []string{"blame", "--porcelain", "-w"} + for _, r := range ranges { + args = append(args, "-L", fmt.Sprintf("%d,+%d", r.start, r.count)) + } + return append(args, rev, "--", file), nil +} + +// blamedLine is one line git blame gives to a commit. +type blamedLine struct{ commit, text string } + +// parseBlame reads git blame --porcelain: a header per line, the commit's +// fields the first time it appears, and the line itself after a tab. +// Anything else is an error, never evidence of absence. +func parseBlame(out string) ([]blamedLine, error) { + var lines []blamedLine + commit := "" + for ln := range strings.Lines(out) { + ln = strings.TrimSuffix(ln, "\n") + switch { + case strings.HasPrefix(ln, "\t"): + if commit == "" { + return nil, errors.New("git blame: a line with no header") + } + lines = append(lines, blamedLine{commit: commit, text: ln[1:]}) + commit = "" + case commit == "": + f := strings.Fields(ln) + if len(f) < 3 || len(f) > 4 || checkSHA(f[0]) != nil { + return nil, fmt.Errorf("git blame: unreadable header %q", terminalSafe(ln)) + } + for _, n := range f[1:] { + if v, err := strconv.Atoi(n); err != nil || v <= 0 { + return nil, fmt.Errorf("git blame: unreadable header %q", terminalSafe(ln)) + } + } + commit = f[0] + } + } + if commit != "" { + return nil, errors.New("git blame: a header with no line") + } + return lines, nil +} + +var generatedMarkerRE = regexp.MustCompile(`Code generated .* DO NOT EDIT`) + +// generatedFiles are written whole by a tool, whatever they hold. +var generatedFiles = []string{"go.sum", "package-lock.json", "yarn.lock", "pnpm-lock.yaml"} + +// cozyRDGenerated are the keys hack/update-crd.sh writes into an +// ApplicationDefinition, the rest of which people write. +var cozyRDGenerated = []string{ + "apiVersion", "kind", "metadata.name", "spec.application.openAPISchema", "spec.release.prefix", + "spec.release.chartRef.kind", "spec.release.chartRef.name", "spec.release.chartRef.namespace", + "spec.dashboard.description", "spec.dashboard.icon", "spec.dashboard.keysOrder", +} + +var yamlKeyRE = regexp.MustCompile(`^( *)([A-Za-z0-9_.-]+):(?:\s|$)`) + +// generatedLines says which lines of a file at a commit a tool wrote: all of +// them for a generated file, the keys hack/update-crd.sh sets for an +// ApplicationDefinition, found by indentation, and none otherwise. ok is +// false for an ApplicationDefinition the scan cannot read, whose lines are +// then not blamed at all. +func generatedLines(file string, blob []byte) (skip map[int]bool, whole, ok bool) { + base := path.Base(file) + if strings.HasPrefix(base, "zz_generated") || slices.Contains(generatedFiles, base) { + return nil, true, true + } + lines := strings.Split(string(blob), "\n") + for _, ln := range lines[:min(len(lines), 100)] { + if generatedMarkerRE.MatchString(ln) { + return nil, true, true + } + } + if rd, _ := path.Match(cozyRDGlob, file); !rd { + return nil, false, true + } + skip = map[int]bool{} + type key struct { + indent int + name string + } + var stack []key + genIndent := -1 + for i, ln := range lines { + n := i + 1 + trimmed := strings.TrimLeft(ln, " ") + indent := len(ln) - len(trimmed) + switch { + case strings.HasPrefix(trimmed, "\t") || strings.HasPrefix(ln, "---") && i > 0: + return nil, false, false + case trimmed == "" || strings.HasPrefix(trimmed, "#"): + if genIndent >= 0 { + skip[n] = true + } + continue + case genIndent >= 0 && indent > genIndent: + skip[n] = true + continue + } + genIndent = -1 + m := yamlKeyRE.FindStringSubmatch(ln) + if m == nil { + continue + } + for len(stack) > 0 && stack[len(stack)-1].indent >= indent { + stack = stack[:len(stack)-1] + } + stack = append(stack, key{indent, m[2]}) + var names []string + for _, k := range stack { + names = append(names, k.name) + } + if slices.Contains(cozyRDGenerated, strings.Join(names, ".")) { + skip[n] = true + genIndent = indent + } + } + return skip, false, true +} + +// trimmedLines is the set of a blob's lines with their edges trimmed, which +// is all a reformat of indentation or trailing space changes. +func trimmedLines(blob []byte) map[string]bool { + set := map[string]bool{} + for ln := range strings.Lines(string(blob)) { + set[strings.TrimSpace(ln)] = true + } + return set +} + +// shown is a path as a diagnostic names it. +func shown(p string) string { return strconv.Quote(p) } + +// incomplete is a diagnostic about evidence the detector could not read. +func incomplete(format string, args ...any) string { + return oneLineText("incomplete evidence: " + fmt.Sprintf(format, args...)) +} + +// dependencies reads what v builds on that target lacks, with what it could +// not read. V1 looks at the paths whose counterpart target lacks, V2c at +// the ones it has. +func (b *batch) dependencies(v *verdict, target string) ([]dependency, []string, error) { + if err := b.load(v); err != nil { + return nil, nil, err + } + if b.brought == nil { + var err error + if b.brought, err = broughtBy(b.base, b.mainHead); err != nil { + return nil, nil, err + } + } + corr, unknown, err := b.correspondence(v, target) + if err != nil { + return nil, nil, err + } + var diag []string + for _, p := range unknown { + diag = append(diag, incomplete("git skipped rename detection for %s", shown(p))) + } + var at []string + for _, c := range corr { + at = append(at, c) + } + have, err := treeEntries(target, at) + if err != nil { + return nil, nil, err + } + parent, err := resolve(v.o.merge + "^1") + if err != nil { + return nil, nil, err + } + byMerge := map[string]*dependency{} + add := func(merge, p string, added bool) { + d := byMerge[merge] + if d == nil { + d = &dependency{merge: merge, v: b.findMerge(merge)} + byMerge[merge] = d + } + if !slices.Contains(d.paths, p) { + d.paths = append(d.paths, p) + } + d.added = d.added || added + } + files, blamed := 0, 0 + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok || c.status != 'M' && c.status != 'D' { + continue + } + if !utf8.ValidString(c.path) { + diag = append(diag, incomplete("a path that is not UTF-8, %s", shown(c.path))) + continue + } + if files++; files > maxDepFiles { + diag = append(diag, incomplete("more than %d files to read", maxDepFiles)) + break + } + cur, present := have[cp] + if !present { + out, err := gitTimed("--literal-pathspecs", "log", "--first-parent", "--diff-merges=first-parent", "--no-renames", "--diff-filter=A", "--format=%H", "--no-patch", "-1", "--end-of-options", b.base+".."+parent, "--", c.path) + switch { + case errors.Is(err, errDeadline): + diag = append(diag, incomplete("%s, finding what added %s", err, shown(c.path))) + continue + case err != nil: + return nil, nil, err + } + adder := strings.TrimSpace(out) + if adder == "" { + diag = append(diag, oneLineText(fmt.Sprintf("%s is not on the line, and nothing merged into main since it branched added it", shown(c.path)))) + continue + } + if checkSHA(adder) != nil { + return nil, nil, fmt.Errorf("git log for %s: %q is not a commit", shown(c.path), terminalSafe(adder)) + } + add(adder, c.path, true) + continue + } + found, more, err := b.blamed(v, parent, c.path, cur, target, maxBlamedLines-blamed) + if err != nil { + return nil, nil, err + } + blamed += more.lines + diag = append(diag, more.diag...) + for merge := range found { + add(merge, c.path, false) + } + if more.capped { + break + } + } + var deps []dependency + for _, d := range byMerge { + slices.Sort(d.paths) + deps = append(deps, *d) + // The rules decide before the line is looked at, so whether the + // line carries a prerequisite they keep out is asked here. + if f := d.v; f != nil && f.rule != 0 && f.rule != ruleOnLine && f.rule != ruleNewer && b.lineHist != nil { + if _, asked := b.lineCarries[f]; !asked { + if err := b.load(f); err != nil { + return nil, nil, err + } + how, err := b.lineHist.carries(f.o, f.commits) + if err != nil { + return nil, nil, err + } + if b.lineCarries == nil { + b.lineCarries = map[*verdict]bool{} + } + b.lineCarries[f] = how != "" + } + } + } + slices.SortFunc(deps, func(x, y dependency) int { return strings.Compare(x.merge, y.merge) }) + return deps, diag, nil +} + +// blameResult is what blaming one file read: how many lines, what it could +// not read, and whether the original's line budget ran out. +type blameResult struct { + lines int + diag []string + capped bool +} + +// blamed gives each line of file that v removes or changes, at parent, to +// the first-parent commit that brought it, and keeps the commits one of +// whose lines target's counterpart of the file, cur, lacks. +func (b *batch) blamed(v *verdict, parent, file string, cur treeEntry, target string, budget int) (map[string]bool, blameResult, error) { + var res blameResult + entries, err := treeEntries(parent, []string{file}) + if err != nil { + return nil, res, err + } + pre := entries[file] + switch { + case pre.mode == "120000" || cur.mode == "120000": + res.diag = append(res.diag, incomplete("%s is a symlink", shown(file))) + return nil, res, nil + case pre.kind == "commit" || cur.kind == "commit": + res.diag = append(res.diag, incomplete("%s is a submodule", shown(file))) + return nil, res, nil + case !pre.regular() || !cur.regular(): + res.diag = append(res.diag, incomplete("%s is no file git can blame", shown(file))) + return nil, res, nil + } + // A file the line already has as the original makes it builds on + // nothing the line lacks, whoever wrote the lines it replaced. + if v.o.merge != "" && cur.oid != pre.oid { + post, err := treeEntries(v.o.merge, []string{file}) + if err != nil { + return nil, res, err + } + if p1, ok := post[file]; ok && p1.regular() { + if cur.oid == p1.oid { + return nil, res, nil + } + applied, _, err := b.alreadyApplied(pre.oid, cur.oid, p1.oid) + if err != nil { + return nil, res, err + } + if applied { + return nil, res, nil + } + } + } + for _, oid := range []string{pre.oid, cur.oid} { + out, err := git("cat-file", "-s", oid) + if err != nil { + return nil, res, err + } + size, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + return nil, res, fmt.Errorf("git cat-file -s %s: %q is no size", oid, terminalSafe(out)) + } + if size > maxDepBlob { + res.diag = append(res.diag, incomplete("%s is larger than %d bytes", shown(file), maxDepBlob)) + return nil, res, nil + } + } + diff, err := gitTimed("--literal-pathspecs", "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "-U0", "--end-of-options", parent, v.o.merge, "--", file) + if errors.Is(err, errDeadline) { + res.diag = append(res.diag, incomplete("%s, diffing %s", err, shown(file))) + return nil, res, nil + } else if err != nil { + return nil, res, err + } + ranges, binary, err := oldRanges(diff) + switch { + case err != nil: + return nil, res, fmt.Errorf("the diff of %s in #%d: %w", shown(file), v.o.number, err) + case binary: + res.diag = append(res.diag, incomplete("%s is binary", shown(file))) + return nil, res, nil + case len(ranges) == 0: + return nil, res, nil + } + blobs, err := readObjects([]string{pre.oid, cur.oid}, "blob") + if err != nil { + return nil, res, err + } + skip, whole, ok := generatedLines(file, blobs[pre.oid]) + switch { + case whole: + return nil, res, nil + case !ok: + res.diag = append(res.diag, incomplete("%s, whose generated keys could not be told apart", shown(file))) + return nil, res, nil + } + var kept []hunk + for _, r := range ranges { + for n := r.start; n < r.start+r.count; n++ { + if skip[n] { + continue + } + if l := len(kept) - 1; l >= 0 && kept[l].start+kept[l].count == n { + kept[l].count++ + } else { + kept = append(kept, hunk{n, 1}) + } + res.lines++ + } + } + if res.lines > budget { + res.diag = append(res.diag, incomplete("more than %d lines to blame", maxBlamedLines)) + res.capped = true + return nil, res, nil + } + if len(kept) == 0 { + return nil, res, nil + } + args, err := blameArgs(parent, file, kept) + if err != nil { + return nil, res, err + } + out, err := gitTimed(args...) + if errors.Is(err, errDeadline) { + res.diag = append(res.diag, incomplete("%s, blaming %s", err, shown(file))) + return nil, res, nil + } else if err != nil { + return nil, res, err + } + lines, err := parseBlame(out) + if err != nil { + return nil, res, fmt.Errorf("blaming %s for #%d: %w", shown(file), v.o.number, err) + } + onTarget := trimmedLines(blobs[cur.oid]) + found := map[string]bool{} + unmapped := false + for _, ln := range lines { + merge, inRange := b.brought[ln.commit] + switch { + case !inRange: + // An ancestor of the line's fork point, which the line has, or + // a commit main does not have at all. + before, err := b.beforeBase(ln.commit) + if err != nil { + return nil, res, err + } + unmapped = unmapped || !before + case merge == v.o.merge: + case !onTarget[strings.TrimSpace(ln.text)]: + found[merge] = true + } + } + if unmapped { + res.diag = append(res.diag, incomplete("git blame gives lines of %s to a commit that did not come through main", shown(file))) + } + return found, res, nil +} + +// timed adds what has passed since start to the time the detector took. +func (b *batch) timed(start time.Time) { b.detecting += time.Since(start) } + +// beforeBase says whether commit is an ancestor of the line's fork point, +// once per commit. +func (b *batch) beforeBase(commit string) (bool, error) { + if ok, seen := b.ancestors[commit]; seen { + return ok, nil + } + ok, err := isAncestor(commit, b.base) + if err == nil { + if b.ancestors == nil { + b.ancestors = map[string]bool{} + } + b.ancestors[commit] = ok + } + return ok, err +} + +// analyze reads what every original the policy takes and nothing carries +// builds on that the line lacks, and keeps it with the original, for the +// parks and waits that follow from it. +func (b *batch) analyze() error { + defer b.timed(time.Now()) + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + deps, diag, err := b.dependencies(v, b.lineHead) + if err != nil { + return err + } + v.deps = deps + v.addDiag(diag...) + } + return nil +} + +// analysisMode is how a classification treats an original that changes a +// workflow: as the scheduled run does, whose token cannot push one, or as a +// maintainer's run. +type analysisMode struct{ noWorkflowFiles bool } + +// edgeClass is what a prerequisite is to an original that builds on it, +// from what never matters to what decides first. +type edgeClass int + +const ( + edgeSatisfied edgeClass = iota + // edgeNewer is held by rule 6 on this line. + edgeNewer + // edgeDeferred is a deferred conflict, or deferred for a reason. + edgeDeferred + // edgeWorkflow waits on the open batch for a maintainer. + edgeWorkflow + // edgeMissing is on neither the line, the batch nor the record. + edgeMissing + // edgeNever never reaches the line. + edgeNever +) + +// edge is a dependency whose prerequisite has not reached the line, with +// what that prerequisite is. +type edge struct { + dep dependency + class edgeClass +} + +// ruleClass is what the rules alone make of a prerequisite, or ok false when +// it depends on what this classification makes of it: one the line or the +// open batch carries is there, whatever its label, drop or rule says, and a +// direct push and an original the rules keep out never come. +func (b *batch) ruleClass(f *verdict) (edgeClass, bool) { + switch { + case f == nil: + return edgeNever, true + case f.rule == ruleOnLine || f.onBatch != "" || f.batched != "" || b.lineCarries[f]: + return edgeSatisfied, true + case f.rule == ruleNewer: + return edgeNewer, true + case f.rule != 0: + return edgeNever, true + } + return 0, false +} + +// edges sorts dependencies by what classOf says of each prerequisite into +// those that never come and those still to come, dropping the ones there. +func edges(deps []dependency, classOf func(*verdict) edgeClass) (never, pending []edge) { + for _, d := range deps { + switch c := classOf(d.v); c { + case edgeSatisfied: + case edgeNever: + never = append(never, edge{d, c}) + default: + pending = append(pending, edge{d, c}) + } + } + return never, pending +} + +// strongest is the class of prerequisite that decides a wait, and the +// prerequisites of that class, in merge order. +func strongest(pending []edge) (edgeClass, []*verdict) { + top := edgeSatisfied + for _, e := range pending { + top = max(top, e.class) + } + var out []*verdict + for _, e := range pending { + if e.class == top && !slices.Contains(out, e.dep.v) { + out = append(out, e.dep.v) + } + } + return top, out +} + +// auxiliary says whether a path holds tests or documentation, which an +// original can change without building on what it needs to work. +func auxiliary(p string) bool { + parts := strings.Split(p, "/") + base := parts[len(parts)-1] + for _, d := range parts[:len(parts)-1] { + if d == "test" || d == "tests" || d == "testdata" { + return true + } + } + switch { + case strings.HasSuffix(base, "_test.go"), strings.Contains(base, ".test."), strings.Contains(base, ".spec."), strings.HasSuffix(base, ".bats"): + return true + case len(parts) > 1 && parts[0] == "docs": + return true + } + return len(parts) == 1 && strings.HasSuffix(base, ".md") +} + +// workflowPrimary says whether v waits for a maintainer rather than parks: +// in the scheduled mode, an original that changes a workflow, all of whose +// missing dependencies that never come are tests or docs, and whose +// workflow lines build on nothing still missing. The workflow is what it is +// for, and a maintainer brings over what applies. Only what is still +// missing counts: never and pending are what is left once what is there was +// taken out. +func workflowPrimary(mode analysisMode, v *verdict, never, pending []edge) (bool, error) { + if !mode.noWorkflowFiles || len(never) == 0 { + return false, nil + } + for _, e := range never { + if slices.ContainsFunc(e.dep.paths, func(p string) bool { return !auxiliary(p) }) { + return false, nil + } + } + for _, e := range append(slices.Clone(never), pending...) { + if slices.ContainsFunc(e.dep.paths, func(p string) bool { return strings.HasPrefix(p, workflowsDir) }) { + return false, nil + } + } + files, err := workflowFiles(v.commits) + return len(files) > 0, err +} + +// testsOnly is the note a workflow original gets that waits for a +// maintainer although its tests build on what never comes. +func testsOnly(v *verdict, never []edge, l line) string { + var who []string + for _, e := range never { + if e.dep.v != nil { + who = append(who, fmt.Sprintf("#%d", e.dep.v.o.number)) + } else { + who = append(who, e.dep.merge[:12]) + } + } + slices.Sort(who) + return oneLineText(fmt.Sprintf("only its tests or docs build on %s, which never reaches %s", strings.Join(slices.Compact(who), ", "), l.branch())) +} + +// dependencyPark is the park of v for prerequisites that never come. +func dependencyPark(v *verdict, never []edge) *parkedEntry { + p := &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkDependency} + for _, e := range never { + if e.dep.v != nil { + p.Needs = append(p.Needs, e.dep.v.o.number) + } else { + p.Commits = append(p.Commits, e.dep.merge) + } + } + slices.Sort(p.Needs) + p.Needs = slices.Compact(p.Needs) + slices.Sort(p.Commits) + p.Commits = slices.Compact(p.Commits) + return p +} + +// depsAt is what v builds on that target lacks, read once per target: the +// run asks again at its head once earlier picks moved the branch. +func (b *batch) depsAt(v *verdict, target string) ([]dependency, error) { + key := target + " " + v.o.merge + if d, ok := b.depCache[key]; ok { + return d, nil + } + deps, diag, err := b.dependencies(v, target) + if err != nil { + return nil, err + } + v.addDiag(diag...) + if b.depCache == nil { + b.depCache = map[string][]dependency{} + } + b.depCache[key] = deps + return deps, nil +} + +// stateKind is what a classification makes of an original the policy takes +// that nothing carries, in the order that decides it. +type stateKind int + +const ( + // stHeld is deferred for a reason a person gave. + stHeld stateKind = iota + 1 + // stPartial would leave part of it without effect on the line. + stPartial + // stWorkflowPrimary waits for a maintainer, though its tests build on + // what never comes. + stWorkflowPrimary + // stParked builds on what never comes. + stParked + // stNeeds waits on prerequisites still to come. + stNeeds + // stOwn is what the record says of it, or missing. + stOwn +) + +// state is a classification of one original: what it is, what it is to an +// original that builds on it, and for a wait the class that decides it and +// the prerequisites of that class. +type state struct { + kind stateKind + class edgeClass + wait edgeClass + needs []*verdict +} + +// classify decides, in merge order, what becomes of every original the +// policy takes that neither the line nor the open batch carries, as an +// operation that picks nothing sees it: check, and the ones that only +// change the record. held are the reasons people deferred originals for, +// recorded what the record defers, and listed says the record's workflow +// waits still wait on an open batch. Parks it finds go into each verdict. +func (b *batch) classify(mode analysisMode, held map[int]string, recorded map[int]deferredEntry, listed bool) (map[*verdict]*state, error) { + states := map[*verdict]*state{} + classOf := func(f *verdict) edgeClass { + if c, ok := b.ruleClass(f); ok { + return c + } + if s := states[f]; s != nil { + return s.class + } + return edgeMissing + } + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + s := &state{} + states[v] = s + e := recorded[v.o.number] + never, pending := edges(v.deps, classOf) + primary, err := workflowPrimary(mode, v, never, pending) + if err != nil { + return nil, err + } + switch { + case held[v.o.number] != "": + s.kind, s.class = stHeld, edgeDeferred + case v.park != nil && v.park.Kind == parkPartial: + s.kind, s.class = stPartial, edgeNever + case primary: + s.kind, s.class = stWorkflowPrimary, edgeMissing + if listed && e.Workflows { + s.class = edgeWorkflow + } + case len(never) > 0: + v.park = dependencyPark(v, never) + s.kind, s.class = stParked, edgeNever + case len(pending) > 0: + s.kind = stNeeds + s.wait, s.needs = strongest(pending) + s.class = s.wait + case listed && e.Workflows: + s.kind, s.class = stOwn, edgeWorkflow + case !e.Workflows && len(e.Needs) == 0 && (e.Commit != "" || len(e.Files) > 0): + s.kind, s.class = stOwn, edgeDeferred + default: + s.kind, s.class = stOwn, edgeMissing + } + } + return states, nil +} diff --git a/cmd/backport-batch/deps_test.go b/cmd/backport-batch/deps_test.go new file mode 100644 index 000000000..d10088e4b --- /dev/null +++ b/cmd/backport-batch/deps_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// depsOf reads what original n builds on that the line lacks. +func depsOf(t testing.TB, f *fixture, n int) ([]dependency, []string) { + t.Helper() + f.git("fetch", "-q", "origin") + b, err := shapeOf(line11) + if err != nil { + t.Fatal(err) + } + deps, diag, err := b.dependencies(b.find(n), b.lineHead) + if err != nil { + t.Fatal(err) + } + return deps, diag +} + +// depNames lists dependencies as #N or a short commit, with their paths. +func depNames(deps []dependency) []string { + var out []string + for _, d := range deps { + who := d.merge[:12] + if d.v != nil { + who = fmt.Sprintf("#%d", d.v.o.number) + } + out = append(out, fmt.Sprintf("%s %v %v", who, d.paths, d.added)) + } + return out +} + +// Every commit main gained since the line branched is brought by one commit +// of its first-parent line: a pull request's own commits by its merge, a +// direct push by itself, and a commit from before the fork point by none. +func TestBroughtBy(t *testing.T) { + f := newFixture(t) + f.lines() + base := f.git("rev-parse", "origin/release-1.1") + mc3 := f.prCommits(3, "feat(app): three", "", + change{files: map[string]string{"app/3.txt": "3\n"}}, + change{mergeMain: true}, + change{files: map[string]string{"app/3.txt": "3b\n"}}) + f.write("app/push.txt", "pushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: a direct push") + f.git("push", "-q", "origin", "main") + push := f.git("rev-parse", "HEAD") + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + by, err := broughtBy(base, mc5) + if err != nil { + t.Fatal(err) + } + moved := f.git("rev-parse", mc3+"^1") + for _, c := range strings.Fields(f.git("rev-list", mc3+"^1.."+mc3+"^2")) { + if c == moved { + continue + } + if by[c] != mc3 { + t.Errorf("%.12s, a commit of #3's branch, is brought by %.12s, want #3's merge", c, by[c]) + } + } + for c, want := range map[string]string{mc3: mc3, moved: moved, push: push, mc5: mc5, f.git("rev-parse", mc5+"^2"): mc5} { + if by[c] != want { + t.Errorf("%.12s is brought by %.12s, want %.12s", c, by[c], want) + } + } + if _, ok := by[base]; ok { + t.Errorf("the fork point %.12s is in the map", base) + } +} + +// V1: a file the original changes that the line does not have names the +// first-parent commit on main that added it, a merge or a direct push. +func TestDependenciesV1(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.write("app/y.txt", appFile) + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: y, pushed") + f.git("push", "-q", "origin", "main") + push := f.git("rev-parse", "HEAD") + f.pr(5, "fix(app): fix x and y", "", map[string]string{"app/x.txt": "fixed\n", "app/y.txt": "fixed\n"}) + deps, diag := depsOf(t, f, 5) + want := []string{push[:12] + " [app/y.txt] true", "#3 [app/x.txt] true"} + slices.Sort(want) + got := depNames(deps) + slices.Sort(got) + if !slices.Equal(got, want) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want %q", got, diag, want) + } +} + +// v2World gives the line app/a.txt as the base had it, and main a feat that +// changes its second line and a fix of that same line. +func v2World(t testing.TB, onLine string) *fixture { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + if onLine != "" { + f.onLine("release-1.1", "fix(app): a, by hand", map[string]string{"app/a.txt": onLine}) + } + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + return f +} + +// V2c: a line the original changes that blame gives to another original is +// a dependency only while the line's file lacks that line's text. +func TestDependenciesV2c(t *testing.T) { + if deps, diag := depsOf(t, v2World(t, ""), 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want #3 through app/a.txt", depNames(deps), diag) + } + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", "TWO", 1)), 5); len(deps) != 0 { + t.Errorf("dependencies with #3's line on the line by hand = %q; want none", depNames(deps)) + } +} + +// The content check trims a line's edges and nothing else: a reindented line +// is there, a line with a space inside it is not. +func TestDependenciesTrimOnlyEdges(t *testing.T) { + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", " TWO\t", 1)), 5); len(deps) != 0 { + t.Errorf("dependencies with #3's line reindented = %q; want none", depNames(deps)) + } + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", "T WO", 1)), 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) { + t.Errorf("dependencies with a space inside #3's line = %q; want #3", depNames(deps)) + } +} + +// Lines a tool wrote are not blamed: a generated file whole, and the keys +// hack/update-crd.sh writes into an ApplicationDefinition, while the keys a +// person writes there are. +func TestDependenciesSkipGeneratedRanges(t *testing.T) { + const rd = "packages/system/app-rd/cozyrds/app.yaml" + rdFile := func(schema, category string) string { + return "apiVersion: cozystack.io/v1alpha1\nkind: ApplicationDefinition\nmetadata:\n name: app\nspec:\n application:\n kind: App\n openAPISchema: |-\n {\"v\":" + schema + "}\n dashboard:\n category: " + category + "\n keysOrder: [[\"spec\", \"" + schema + "\"]]\n" + } + world := func(t *testing.T, before, feat, fix map[string]string) *fixture { + f := newFixture(t) + f.squash(2, "chore: generated files", "", before) + f.lines() + f.pr(3, "feat(api): regenerate", "", feat) + f.pr(5, "fix(api): regenerate again", "", fix) + return f + } + gen := "// Code generated by controller-gen. DO NOT EDIT.\n\npackage v1\n\nvar a = 1\n" + f := world(t, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 1\n", "api/types_gen.go": gen, rd: rdFile("1", "A")}, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 2\n", "api/types_gen.go": strings.Replace(gen, "= 1", "= 2", 1), rd: rdFile("2", "A")}, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 3\n", "api/types_gen.go": strings.Replace(gen, "= 1", "= 3", 1), rd: rdFile("3", "A")}) + if deps, diag := depsOf(t, f, 5); len(deps) != 0 || len(diag) != 0 { + t.Errorf("dependencies through generated lines = %q, %q; want none", depNames(deps), diag) + } + f = world(t, map[string]string{rd: rdFile("1", "A")}, map[string]string{rd: rdFile("2", "B")}, map[string]string{rd: rdFile("3", "C")}) + if deps, _ := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [" + rd + "] false"}) { + t.Errorf("dependencies through a key a person writes = %q; want #3", depNames(deps)) + } +} + +// Evidence the detector cannot read, a binary file or a symlink, is a +// diagnostic and never a dependency. +func TestDependenciesIncompleteNeverParks(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: an image", "", map[string]string{"app/a.bin": "a\x00one"}) + f.lines() + f.pr(3, "feat(app): a new image", "", map[string]string{"app/a.bin": "a\x00two"}) + f.pr(5, "fix(app): a fixed image", "", map[string]string{"app/a.bin": "a\x00three"}) + if deps, diag := depsOf(t, f, 5); len(deps) != 0 || !slices.Equal(diag, []string{`incomplete evidence: "app/a.bin" is binary`}) { + t.Errorf("dependencies = %q, %q; want none and a diagnostic", depNames(deps), diag) + } +} + +// Output of git the detector cannot read whole is an error, never taken +// for an absent dependency. +func TestDependenciesMalformedOutputIsAnError(t *testing.T) { + sha := strings.Repeat("a", 40) + for name, out := range map[string]string{ + "a short header": sha + " 1\n\tline\n", + "a header of no commit": "aaaa 1 1 1\n\tline\n", + "a header of no number": sha + " x 1 1\n\tline\n", + "a line with no header": "\tline\n", + "a header with no line": sha + " 1 1 1\nauthor x\n", + } { + if _, err := parseBlame(out); err == nil { + t.Errorf("%s: parseBlame accepted %q", name, out) + } + } + if lines, err := parseBlame(sha + " 2 2 1\nauthor x\nfilename a\n\ttwo\n" + sha + " 4 4\n\tfour\n"); err != nil || len(lines) != 2 || lines[1].text != "four" { + t.Errorf("parseBlame of sound output = %+v, %v", lines, err) + } + if _, _, err := oldRanges("@@ -x +1 @@\n"); err == nil { + t.Error("oldRanges accepted an unreadable hunk header") + } +} + +// Paths go to git as they are: a name a pathspec would read as a pattern +// means only itself. +func TestDependenciesLiteralPaths(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): a bracketed file", "", map[string]string{"app/[x].txt": appFile}) + f.pr(4, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix the bracketed file", "", map[string]string{"app/[x].txt": "fixed\n"}) + if deps, _ := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [app/[x].txt] true"}) { + t.Errorf("dependencies = %q; want #3 alone", depNames(deps)) + } +} + +// A file with no final newline blames its last line like any other. +func TestDependenciesNoFinalNewline(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": "one\ntwo"}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": "one\nTWO"}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": "one\nTWO!"}) + if deps, diag := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want #3", depNames(deps), diag) + } +} + +// Blame takes the revision as a full commit name, which git cannot read as +// an option, and refuses --end-of-options, so the tool passes none. +func TestBlameInvocation(t *testing.T) { + sha := strings.Repeat("a", 40) + args, err := blameArgs(sha, "-x", []hunk{{2, 3}, {9, 1}}) + if want := []string{"blame", "--porcelain", "-w", "-L", "2,+3", "-L", "9,+1", sha, "--", "-x"}; err != nil || !slices.Equal(args, want) { + t.Errorf("blameArgs = %q, %v; want %q", args, err, want) + } + for _, rev := range []string{"HEAD", "--output=x", strings.Repeat("a", 39)} { + if _, err := blameArgs(rev, "a", nil); err == nil { + t.Errorf("blameArgs took %q for a revision", rev) + } + } +} + +// What the detector could not read shows with the original whatever became +// of it: picked, waiting for a maintainer, or a deferred conflict, in the +// body and in the report. +func TestDiagnosticsShowOnEveryOutcome(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: images", "", map[string]string{"app/p.bin": "p\x00one", "app/w.bin": "w\x00one", "app/c.bin": "c\x00one", "app/c.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): c, on the line", map[string]string{"app/c.txt": "line\n"}) + f.pr(5, "fix(app): picked", "", map[string]string{"app/p.bin": "p\x00two"}) + f.pr(6, "fix(ci): waiting", "", map[string]string{"app/w.bin": "w\x00two", ".github/workflows/six.yaml": "on: push\n"}) + f.pr(7, "fix(app): conflicting", "", map[string]string{"app/c.bin": "c\x00two", "app/c.txt": "seven\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + for _, s := range []struct{ section, file string }{{"[#5]", "p.bin"}, {"### Deferred conflicts", "c.bin"}, {"### Waiting for a maintainer", "w.bin"}} { + _, after, ok := strings.Cut(stdout, s.section) + if want := " - `incomplete evidence: \"app/" + s.file + "\" is binary`"; !ok || !strings.Contains(after, want) { + t.Errorf("the body does not show %q after %s:\n%s", want, s.section, stdout) + } + } + for n, file := range map[int]string{5: "p.bin", 6: "w.bin", 7: "c.bin"} { + if want := fmt.Sprintf(`"%d":["incomplete evidence: \"app/%s\" is binary"]`, n, file); !strings.Contains(stdout, want) { + t.Errorf("the report lacks %s:\n%s", want, stdout) + } + } +} + +// BenchmarkAnalyze reads what forty originals build on, over a history +// where half of them build on a feat the line lacks. +func BenchmarkAnalyze(b *testing.B) { + f := newFixture(b) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + for n := 10; n < 50; n++ { + file := fmt.Sprintf("app/f%d.txt", n) + if n%2 == 0 { + f.pr(n, "feat(app): "+file, "", map[string]string{file: appFile}) + continue + } + prev := fmt.Sprintf("app/f%d.txt", n-1) + f.pr(n, "fix(app): "+prev, "", map[string]string{prev: strings.Replace(appFile, "two", fmt.Sprint(n), 1), "app/a.txt": strings.Replace(appFile, "three", fmt.Sprint(n), 1)}) + } + f.git("fetch", "-q", "origin") + b.ResetTimer() + for range b.N { + bt, err := shapeOf(line11) + if err != nil { + b.Fatal(err) + } + if err := bt.analyze(); err != nil { + b.Fatal(err) + } + } +} + +// Files the line lacks count toward the files an original may have read +// for it, like the ones blamed. +func TestDependenciesCapMissingFiles(t *testing.T) { + f := newFixture(t) + f.lines() + added, fixed := map[string]string{}, map[string]string{} + for i := range maxDepFiles + 1 { + added[fmt.Sprintf("app/f%03d.txt", i)] = "v1\n" + fixed[fmt.Sprintf("app/f%03d.txt", i)] = "v2\n" + } + f.pr(3, "feat(app): many files", "", added) + f.pr(5, "fix(app): all of them", "", fixed) + if _, diag := depsOf(t, f, 5); !slices.ContainsFunc(diag, func(d string) bool { return strings.Contains(d, fmt.Sprintf("more than %d files", maxDepFiles)) }) { + t.Errorf("diag = %q, want the file cap", diag) + } +} diff --git a/cmd/backport-batch/drop.go b/cmd/backport-batch/drop.go new file mode 100644 index 000000000..1af2e5b3a --- /dev/null +++ b/cmd/backport-batch/drop.go @@ -0,0 +1,304 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "cmp" + "fmt" + "maps" + "slices" + "strings" + "time" +) + +// A drop is a maintainer's decision that an original stays off one line, +// for a reason, where the rules would take it. It is kept in the line's +// record and carried from batch to batch, bound to the original's merge +// commit on main, so a record cannot point it at another original. It is +// written only to the open batch's record, like everything else there. + +// onLine says whether the line carries v, whatever rule decided it. +func (b *batch) onLine(v *verdict) (bool, error) { + if v.rule == ruleOnLine { + return true, nil + } + if err := b.load(v); err != nil { + return false, err + } + how, err := b.lineHist.carries(v.o, v.commits) + return how != "", err +} + +// carried says whether the open batch has a unit of v, by the mapping the +// rebuild takes units off with: found needs every commit of a merged +// original, and one whose pick came out empty is missing from it. +func (b *batch) carried(v *verdict) (bool, error) { + units, err := batchUnits(b.prHead, b.lineHead) + if err != nil { + return false, err + } + for _, u := range units { + if o := b.unitOriginal(u); o != nil && o.o.number == v.o.number { + return true, nil + } + } + return false, nil +} + +// redrop drops the originals in numbers from the line, for reason, or with +// undrop withdraws their drops. What the open batch carries of a dropped +// original comes off it, as --defer takes it off; otherwise only the record +// changes, and the batch stays as it is even when the line moved since. +func (a *app) redrop(b *batch, numbers []int, reason string, undrop bool, push bool) int { + if !b.src.open { + return a.fail(fmt.Errorf("no open batch pull request on %s carries a record; a drop is recorded on one", b.line.branch())) + } + drops := map[int]droppedEntry{} + for _, d := range b.src.rec.Dropped { + drops[d.Original] = d + } + mode, today := modeDrop, a.now().UTC().Format(time.DateOnly) + if undrop { + mode = modeUndrop + } + for _, n := range numbers { + v := b.find(n) + _, isDropped := drops[n] + switch { + case undrop && !isDropped: + return a.fail(fmt.Errorf("#%d is not dropped from %s", n, b.line.branch())) + case undrop: + delete(drops, n) + case v == nil: + return a.fail(fmt.Errorf("#%d did not merge into main after %s branched", n, b.line.branch())) + default: + // backport/skip and rules 1 to 4 decide before the line is + // looked at, so whether it carries the original is asked here. + on, err := b.onLine(v) + if err != nil { + return a.fail(err) + } + if on { + return a.fail(fmt.Errorf("#%d is already on %s; a drop cannot remove code the line has", n, b.line.branch())) + } + drops[n] = droppedEntry{Original: n, Merge: v.o.merge, Reason: reason, On: b.pr.Number, Date: today} + } + } + active := map[int]droppedEntry{} + for n, d := range drops { + if b.find(n) != nil { + active[n] = d + } + } + b.drops = active + out := map[int]string{} + for _, n := range numbers { + v := b.find(n) + if v == nil { + continue + } + if err := b.judge(v, active); err != nil { + return a.fail(err) + } + if err := b.load(v); err != nil { + return a.fail(err) + } + if err := b.rejudged(v); err != nil { + return a.fail(err) + } + if undrop { + continue + } + on, err := b.carried(v) + if err != nil { + return a.fail(err) + } + if on { + out[n] = "dropped" + } + } + hold := map[int]string{} + for n, e := range b.recorded() { + if e.Reason != "" { + hold[n] = e.Reason + } + } + list := slices.SortedFunc(maps.Values(drops), func(x, y droppedEntry) int { return cmp.Compare(x.Original, y.Original) }) + if len(out) > 0 { + maps.Copy(out, hold) + return a.reassemble(b, out, hold, list, mode, push) + } + if err := a.warnReset(b.pr.Head.Ref, b.prHead); err != nil { + return a.fail(err) + } + if _, err := git("switch", "--quiet", "--no-track", "-C", b.pr.Head.Ref, "--end-of-options", b.prHead); err != nil { + return a.fail(err) + } + return a.reclassify(b, b.picks, hold, list, mode, push) +} + +// bindDrops holds each drop the record keeps to the original it names: a +// drop of an original merged since the line branched has to name that +// original's merge commit, and one of an earlier original a merge of it on +// main's first-parent line. The first apply; the others are kept and carried +// on, and decide nothing. +func (b *batch) bindDrops(merges []*original) (map[int]droppedEntry, error) { + return bindEntries(b.src.rec.dropped(), merges, b.mainHead, func(d droppedEntry) string { + return fmt.Sprintf("batch pull request #%d drops #%d by %.12s", b.src.carrier.Number, d.Original, d.Merge) + }) +} + +// waitOrder refuses a record whose waits or parks need an original merged +// after the one that needs it: a dependency is always on main first. +func (b *batch) waitOrder(merges []*original) error { + if b.src.rec == nil { + return nil + } + at := map[int]int{} + for i, o := range merges { + at[o.number] = i + } + check := func(n int, needs []int) error { + i, ok := at[n] + for _, f := range needs { + if j, known := at[f]; ok && known && j >= i { + return fmt.Errorf("batch pull request #%d has #%d wait on #%d, which merged after it", b.src.carrier.Number, n, f) + } + } + return nil + } + for _, d := range b.src.rec.Deferred { + if err := check(d.Original, d.Needs); err != nil { + return err + } + } + for _, p := range b.src.rec.Parked { + if err := check(p.Original, p.Needs); err != nil { + return err + } + } + return nil +} + +// bindEntries binds drops to main: one whose original is among merges, the +// originals main merged since the line branched, has to name that +// original's merge exactly; any other has to be a merge of its number on +// main's first-parent line up to mainHead, as an inherited drop is. It +// returns the drops whose originals are among merges; about says what an +// error is about. +func bindEntries(drops []droppedEntry, merges []*original, mainHead string, about func(droppedEntry) string) (map[int]droppedEntry, error) { + byMerge, byNumber := map[string]*original{}, map[int]*original{} + for _, o := range merges { + byMerge[o.merge], byNumber[o.number] = o, o + } + active := map[int]droppedEntry{} + for _, d := range drops { + at := about(d) + if o := byMerge[d.Merge]; o != nil { + if o.number != d.Original { + return nil, fmt.Errorf("%s, which merged #%d", at, o.number) + } + active[d.Original] = d + continue + } + if o := byNumber[d.Original]; o != nil { + return nil, fmt.Errorf("%s, but #%d merged as %.12s", at, d.Original, o.merge) + } + ok, err := mergedOnMain(d.Merge, mainHead, d.Original) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%s, which is no merge of it on main", at) + } + } + return active, nil +} + +// bindDrops binds each drop a report keeps to main exactly as the record's +// drops are bound, over the window the report classified: main and the +// line at the heads it was made from. Its errors read after "report ". +func (d *reportDoc) bindDrops() error { + if len(d.Overlays.Dropped) == 0 { + return nil + } + for _, h := range []struct{ name, head string }{{"main", d.Built.Main}, {d.Line, d.Built.Line}} { + have, err := lookup(h.head) + if err != nil { + return err + } + if have == "" { + return fmt.Errorf("was made on %s at %.12s, which this clone does not have; fetch %s and run again", h.name, h.head, h.name) + } + } + out, err := git("merge-base", "--end-of-options", d.Built.Main, d.Built.Line) + if err != nil { + return fmt.Errorf("was made from a line that shares no history with main: %w", err) + } + merges, _, err := mainMerges(strings.TrimSpace(out), d.Built.Main) + if err != nil { + return err + } + active, err := bindEntries(d.Overlays.Dropped, merges, d.Built.Main, func(e droppedEntry) string { + return fmt.Sprintf("drops #%d by %.12s", e.Original, e.Merge) + }) + if err != nil { + return err + } + // A drop decides only for an original the report classified, and only + // once every earlier rule has passed it by; an inherited one decides + // nothing. + where, err := d.categories() + if err != nil { + return err + } + for _, n := range d.Primary.Skipped.Dropped { + if _, ok := active[n]; !ok { + return fmt.Errorf("has #%d dropped, but no drop of it binds to an original it classified", n) + } + } + for n := range active { + switch c := where[n]; c { + case "skipped.dropped", "skipped.skip-label", "skipped.before-start", "skipped.type", "skipped.breaking", "skipped.touches", "skipped.on-line": + default: + return fmt.Errorf("drops #%d, which it has %s, though a drop decides before that", n, or(c, "nowhere")) + } + } + return nil +} + +// mergedOnMain says whether merge is on main's first-parent line up to +// mainHead and its message names pull request n, as mainMerges reads one. +func mergedOnMain(merge, mainHead string, n int) (bool, error) { + have, err := lookup(merge) + if err != nil || have == "" { + return false, err + } + if ok, err := isAncestor(merge, mainHead); err != nil || !ok { + return false, err + } + line, err := revList("--first-parent", "--end-of-options", mainHead) + if err != nil || !slices.Contains(line, merge) { + return false, err + } + msgs, err := commitMessages([]string{merge}) + if err != nil { + return false, err + } + number, _, _ := mergedPR(msgs[merge].message) + return number == n, nil +} diff --git a/cmd/backport-batch/drop_test.go b/cmd/backport-batch/drop_test.go new file mode 100644 index 000000000..f88b2d21c --- /dev/null +++ b/cmd/backport-batch/drop_test.go @@ -0,0 +1,576 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// recordOf is the record the open batch #90 carries now. +func recordOf(t *testing.T, gh *fakeGH) *lineRecord { + t.Helper() + rec, err := parseRecord(gh.find(90).Body, notifyLine) + if err != nil || rec == nil { + t.Fatalf("#90 carries no readable record: %v", err) + } + return rec +} + +func dropped(rec *lineRecord) []int { + var out []int + for _, d := range rec.Dropped { + out = append(out, d.Original) + } + return out +} + +// A drop takes a picked original off the batch and keeps it off the line: +// a run that continues the batch does not pick it, and nor does the next +// batch, which carries the drop over from the merged one. +func TestDropTakesAnOriginalOffTheLine(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + mc6 := f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + stdout := mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "6", "--reason", "needs the rework of #4", "--push") + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): five (#5)"}) { + t.Errorf("after the drop the batch carries %q, want #5 alone", got) + } + rec := recordOf(t, gh) + if want := (droppedEntry{Original: 6, Merge: mc6, Reason: "needs the rework of #4", On: 90, Date: "2026-10-26"}); len(rec.Dropped) != 1 || rec.Dropped[0] != want || rec.Built.Options.Mode != modeDrop { + t.Errorf("the record drops %+v (%s), want %+v", rec.Dropped, rec.Built.Options.Mode, want) + } + if s := bodyReportOf(t, stdout); !slices.Equal(s.doc.Primary.Skipped.Dropped, []int{6}) || !slices.Equal(droppedNumbers(s.doc.Overlays.Dropped), []int{6}) { + t.Errorf("the report has %v dropped and %v as the decision", s.doc.Primary.Skipped.Dropped, s.doc.Overlays.Dropped) + } + if !strings.Contains(stdout, "
Dropped from `release-1.1` by a maintainer") { + t.Errorf("the body does not list #6 as dropped:\n%s", stdout) + } + code, _, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stderr, "note: #6 is dropped from release-1.1 on #90 (needs the rework of #4): https://github.com/cozystack/cozystack/pull/6 ") { + t.Errorf("check --cut = %d, want #5 to fail it and a note on #6:\n%s", code, stderr) + } + + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): five (#5)", "fix(app): seven (#7)"}) || !slices.Equal(dropped(recordOf(t, gh)), []int{6}) { + t.Errorf("the continued batch carries %q and drops %v", got, dropped(recordOf(t, gh))) + } + + mergeBatch(f, gh) + f.now = f.now.AddDate(0, 0, 3) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.nextPR = 91 + stdout = mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + rec, err := parseRecord(stdout, notifyLine) + if got := f.subjects("origin/release-1.1", f.remoteHead("backport-batch/release-1.1-20261029")); err != nil || !slices.Equal(got, []string{"fix(app): eight (#8)"}) || !slices.Equal(dropped(rec), []int{6}) { + t.Errorf("the next batch carries %q and drops %v (%v); want #8 alone, #6 still dropped", got, rec, err) + } +} + +// Dropping an original the batch does not carry changes only the record: +// the batch is not rebuilt, so a line that moved since does not move it. +func TestDropIsRecordOnlyWhenTheLineMoved(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only seven", map[string]string{"app/seven.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if recordOf(t, gh).deferred()[7].Commit == "" { + t.Fatal("#7 is not a recorded conflict") + } + f.onLine("release-1.1", "fix(chart): a line-only value", map[string]string{"chart/values.yaml": "replicas: 3\nimage: app:1\n"}) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "7", "--reason", "needs #4", "--push") + rec := recordOf(t, gh) + if f.remoteHead(batchBranch1) != head || !slices.Equal(dropped(rec), []int{7}) || len(rec.deferred()) != 0 || len(gh.edited) != 1 { + t.Errorf("the batch moved to %.12s (from %.12s), the record drops %v and defers %v; want only the record changed", f.remoteHead(batchBranch1), head, dropped(rec), rec.deferred()) + } +} + +// A drop is written to the open batch's record, and to no other: with no +// batch open, or none ever, it is refused and nothing is written. +func TestDropNeedsAnOpenBatch(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only seven", map[string]string{"app/seven.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--drop", "5", "--reason", "r", "--push"); code != 2 || !strings.Contains(stderr, "no open batch pull request on release-1.1 carries a record; a drop is recorded on one") { + t.Errorf("a drop on a line that never had a batch = %d:\n%s", code, stderr) + } + openBatch(t, f, gh) + mergeBatch(f, gh) + written := len(gh.edited) + len(gh.issueComments) + for _, args := range [][]string{{"--drop", "7", "--reason", "r", "--push"}, {"--undrop", "7", "--push"}} { + if code, _, stderr := f.exec(gh, append([]string{"run", "--line", "release-1.1"}, args...)...); code != 2 || !strings.Contains(stderr, "carries a record; a drop is recorded on one") { + t.Errorf("%q after the batch merged = %d:\n%s", args, code, stderr) + } + } + if len(gh.edited)+len(gh.issueComments) != written { + t.Error("a refused drop wrote to GitHub") + } +} + +// A drop is a decision kept beside whatever else decides an original: the +// rule, label or carry that applies first is its category, and the drop +// stays on record. backport/include does not lift it. +func TestDropOverlay(t *testing.T) { + setup := func(t *testing.T) (*fixture, *fakeGH) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "feat(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{labels: map[string][]int{labelInclude: {7}}} + openBatch(t, f, gh) + return f, gh + } + category := func(t *testing.T, gh *fakeGH, n int) string { + t.Helper() + s := bodyReportOf(t, gh.find(90).Body) + where, err := s.doc.categories() + if err != nil { + t.Fatal(err) + } + return where[n] + } + merge := func(f *fixture, n int) string { + mc, _, _ := strings.Cut(f.git("log", "--first-parent", "--format=%H", "--grep", fmt.Sprintf("(#%d)$", n), "main"), "\n") + return mc + } + drop := func(t *testing.T, f *fixture, gh *fakeGH, n string) { + t.Helper() + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", n, "--reason", "not for 1.1", "--push") + } + t.Run("then backport/skip", func(t *testing.T) { + f, gh := setup(t) + drop(t, f, gh, "6") + gh.labels[labelSkip] = []int{6} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if got := category(t, gh, 6); got != "skipped.skip-label" || !slices.Equal(dropped(recordOf(t, gh)), []int{6}) { + t.Errorf("#6 is %q, dropped %v", got, dropped(recordOf(t, gh))) + } + }) + t.Run("then picked onto the line by hand", func(t *testing.T) { + f, gh := setup(t) + drop(t, f, gh, "6") + pickOnLine(f, 6, map[string]string{"app/six.txt": "6\n"}) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if got := category(t, gh, 6); got != "skipped.on-line" || !slices.Equal(dropped(recordOf(t, gh)), []int{6}) { + t.Errorf("#6 is %q, dropped %v", got, dropped(recordOf(t, gh))) + } + }) + t.Run("a feat backport/include let in", func(t *testing.T) { + f, gh := setup(t) + drop(t, f, gh, "7") + if got := category(t, gh, 7); got != "skipped.dropped" || slices.Contains(f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)), "feat(app): seven (#7)") { + t.Errorf("#7 is %q; want it dropped and off the batch", got) + } + if !strings.Contains(gh.find(90).Body, "`backport/include`, skipped by a maintainer's drop") { + t.Errorf("the overrides do not say what became of #7:\n%s", gh.find(90).Body) + } + }) + t.Run("undropped", func(t *testing.T) { + f, gh := setup(t) + drop(t, f, gh, "6") + head := f.remoteHead(batchBranch1) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--undrop", "6", "--push") + if got := category(t, gh, 6); got != "missing" || len(recordOf(t, gh).Dropped) != 0 || f.remoteHead(batchBranch1) != head || recordOf(t, gh).Built.Options.Mode != modeUndrop { + t.Errorf("#6 is %q, dropped %v; want it missing until the next run, the batch where it was", got, dropped(recordOf(t, gh))) + } + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/6 ") { + t.Errorf("check = %d, want #6 missing:\n%s", code, stdout) + } + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if got := category(t, gh, 6); got != "picked" { + t.Errorf("after the next run #6 is %q, want it picked again", got) + } + }) + t.Run("a new original during a change to the record alone", func(t *testing.T) { + f, gh := setup(t) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + drop(t, f, gh, "7") + mustRun(t, f, gh, "run", "--line", "release-1.1", "--undrop", "7", "--push") + if got := category(t, gh, 8); got != "missing" { + t.Errorf("#8 is %q, want missing until a run tries it", got) + } + }) + // A stored drop is bound to the merge it names: one that names another + // original's, or one of no merge of it, is refused; one of an original + // merged before the line branched is kept and decides nothing. + t.Run("a stored drop of an earlier commit that names the original", func(t *testing.T) { + // Before release-1.1 branched, a commit on main already ended in + // (#6); the drop has to name the merge of #6 the line can take. + f := newFixture(t) + f.release("release-1.0") + f.tag("v1.0.0") + early := f.squash(6, "docs: what six will change", "", map[string]string{"docs/six.md": "6\n"}) + f.release("release-1.1") + f.tag("v1.1.0") + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + drop(t, f, gh, "6") + gh.find(90).Body = strings.Replace(gh.find(90).Body, `{"original":6,"merge":"`+merge(f, 6)+`"`, `{"original":6,"merge":"`+early+`"`, 1) + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "but #6 merged as") { + t.Errorf("check = %d, want 2 naming the merge of #6:\n%s", code, stderr) + } + }) + t.Run("a stored drop of a commit inside a pull request", func(t *testing.T) { + // A commit of a pull request's own branch can end in its number too, + // but only the merge on main's first-parent line is the original. + f := newFixture(t) + f.release("release-1.0") + f.tag("v1.0.0") + mc3 := f.prCommits(3, "feat(app): three", "", change{files: map[string]string{"app/three.txt": "3\n"}, message: "feat(app): the part of three (#3)"}) + inner := f.git("rev-parse", mc3+"^2") + f.release("release-1.1") + f.tag("v1.1.0") + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + drop(t, f, gh, "6") + gh.find(90).Body = strings.Replace(gh.find(90).Body, `{"original":6,"merge":"`+merge(f, 6)+`"`, `{"original":3,"merge":"`+inner+`"`, 1) + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "which is no merge of it on main") { + t.Errorf("check = %d, want 2:\n%s", code, stderr) + } + }) + for _, tc := range []struct { + name string + original func(f *fixture) (int, string) + refused bool + }{ + {"another original's merge", func(f *fixture) (int, string) { return 6, merge(f, 5) }, true}, + {"a commit that merged no pull request", func(f *fixture) (int, string) { return 6, f.git("rev-list", "--max-parents=0", "main") }, true}, + {"a merge from before the line branched, of another pull request", func(f *fixture) (int, string) { return 2, merge(f, 1) }, true}, + {"a merge from before the line branched", func(f *fixture) (int, string) { return 1, merge(f, 1) }, false}, + } { + t.Run("a stored drop of "+tc.name, func(t *testing.T) { + f, gh := setup(t) + drop(t, f, gh, "6") + n, mc := tc.original(f) + gh.find(90).Body = strings.Replace(gh.find(90).Body, `{"original":6,"merge":"`+merge(f, 6)+`"`, fmt.Sprintf(`{"original":%d,"merge":"%s"`, n, mc), 1) + code, _, stderr := f.exec(gh, "check", "--line", "release-1.1") + switch { + case tc.refused && (code != 2 || !strings.Contains(stderr, fmt.Sprintf("drops #%d by %.12s", n, mc))): + t.Errorf("check = %d, want 2 naming the drop:\n%s", code, stderr) + case !tc.refused && code == 2: + t.Errorf("check = %d, want the drop kept:\n%s", code, stderr) + case !tc.refused: + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + if !slices.Equal(dropped(recordOf(t, gh)), []int{1}) || category(t, gh, 6) != "picked" { + t.Errorf("the next run drops %v and puts #6 in %q; want #1's drop carried on and #6 picked", dropped(recordOf(t, gh)), category(t, gh, 6)) + } + } + }) + } +} + +// What a drop cannot do is refused before anything is touched. +func TestDropRefuses(t *testing.T) { + f := newFixture(t) + f.lines() + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{labels: map[string][]int{labelSkip: {7, 9}}} + head := openBatch(t, f, gh) + f.onLine("release-1.1", "fix(app): five\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "5\n"}) + // The line carries these too, by hand, while backport/skip and rule 2 + // decide them before the line is looked at. + mc7 := f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + mc8 := f.pr(8, "feat(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + f.onLine("release-1.1", "fix(app): seven\n\n(cherry picked from commit "+mc7+")", map[string]string{"app/seven.txt": "7\n"}) + f.onLine("release-1.1", "feat(app): eight\n\n(cherry picked from commit "+mc8+")", map[string]string{"app/eight.txt": "8\n"}) + // #9 is on the line only through a pick of its own commit, which a later + // line change has since moved away from what main has. + mc9 := f.pr(9, "fix(app): nine", "", map[string]string{"app/nine.txt": "9\n"}) + f.onLine("release-1.1", "fix(app): nine\n\n(cherry picked from commit "+f.git("rev-parse", mc9+"^2")+")", map[string]string{"app/nine.txt": "9\n"}) + f.onLine("release-1.1", "fix(app): nine, as the line needs it", map[string]string{"app/nine.txt": "nine on release-1.1\n"}) + for _, tc := range []struct { + args []string + why string + }{ + {[]string{"--drop", "4242", "--reason", "r"}, "#4242 did not merge into main"}, + {[]string{"--drop", "5", "--reason", "r"}, "already on release-1.1; a drop cannot remove code the line has"}, + {[]string{"--drop", "7", "--reason", "r"}, "#7 is already on release-1.1; a drop cannot remove code the line has"}, + {[]string{"--drop", "8", "--reason", "r"}, "#8 is already on release-1.1; a drop cannot remove code the line has"}, + {[]string{"--drop", "9", "--reason", "r"}, "#9 is already on release-1.1; a drop cannot remove code the line has"}, + {[]string{"--drop", "6"}, "--drop needs a --reason"}, + {[]string{"--drop", "6", "--reason", "r", "--defer", "6"}, "give one"}, + {[]string{"--drop", "6", "--reason", "r", "--continue"}, "takes no --onto, --continue or --date"}, + {[]string{"--drop", "6", "--reason", "r", "--onto", head}, "takes no --onto, --continue or --date"}, + {[]string{"--drop", "6", "--reason", "r", "--date", "2026-10-26"}, "takes no --onto, --continue or --date"}, + {[]string{"--undrop", "6", "--reason", "r"}, "--undrop takes no --reason"}, + {[]string{"--undrop", "6"}, "#6 is not dropped from release-1.1"}, + {[]string{"--drop", "6", "--undrop", "6", "--reason", "r"}, "give one"}, + {[]string{"--drop", "six", "--reason", "r"}, "not a list of pull request numbers"}, + } { + code, _, stderr := f.exec(gh, append(append([]string{"run", "--line", "release-1.1"}, tc.args...), "--push")...) + f.git("switch", "-q", "main") + if code != 2 || !strings.Contains(stderr, tc.why) { + t.Errorf("run %q = %d, want 2 with %q:\n%s", tc.args, code, tc.why, stderr) + } + } + if f.remoteHead(batchBranch1) != head || len(gh.edited) != 0 || len(reportsPosted(gh)) != 1 { + t.Errorf("a refused drop moved the batch to %.12s or wrote %q", f.remoteHead(batchBranch1), gh.edited) + } +} + +// A drop or an undrop asked for while a close was left changes nothing: the +// run finishes the close, and no open batch pull request is left to record +// it on. +func TestDropDuringAnUnfinishedClose(t *testing.T) { + deferAll := []string{"run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the app e2e", "--push"} + for _, tc := range []struct { + name string + stop func(f *fixture, gh *fakeGH) + refused bool + args []string + }{ + {"a drop after the close failed", func(f *fixture, gh *fakeGH) { gh.fail = map[string]error{"closePull": errString("HTTP 502")} }, false, []string{"--drop", "6", "--reason", "not for 1.1"}}, + {"an undrop after the delete failed", func(f *fixture, gh *fakeGH) { + gh.after = map[string]func(){"closePull": func() { f.refusePushes(true) }} + }, true, []string{"--undrop", "6"}}, + } { + t.Run(tc.name, func(t *testing.T) { + f, gh, _ := closeWorld(t) + tc.stop(f, gh) + if code, _, stderr := f.exec(gh, deferAll...); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if tc.refused { + f.refusePushes(false) + } + gh.after = nil + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + code, _, stderr := f.exec(gh, append([]string{"run", "--line", "release-1.1", "--push"}, tc.args...)...) + f.git("switch", "-q", "main") + rec, err := parseRecord(gh.find(90).Body, notifyLine) + if code != 2 || !strings.Contains(stderr, "finished the close of #90 and made no other change") || gh.find(90).State != "closed" || f.remoteHead(batchBranch1) != "" || err != nil || len(rec.Dropped) != 0 { + t.Errorf("run %q = %d, #90 %s, branch %q, record %+v, %v; want 2, the close finished and nothing dropped:\n%s", tc.args, code, gh.find(90).State, f.remoteHead(batchBranch1), rec, err, stderr) + } + }) + } +} + +// The batch carries an original whose first commit the line already had, so +// its pick came out empty and its unit holds the rest: a drop still takes +// that unit off. +func TestDropTakesOffAUnitWithAnEmptyPick(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): the first part, on its own", map[string]string{"app/already.txt": "first\n"}) + f.prCommits(5, "fix(app): a fix in two parts", "", + change{files: map[string]string{"app/already.txt": "first\n"}}, + change{files: map[string]string{"app/new.txt": "second\n"}}, + ) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "5", "--reason", "not for 1.1", "--push") + head := f.remoteHead(batchBranch1) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): six (#6)"}) || f.git("ls-tree", "--name-only", head, "--", "app/new.txt") != "" { + t.Errorf("after the drop the batch carries %q, want #6 alone and nothing of #5", got) + } +} + +// A report keeps each drop the record keeps, with the merge commit it is +// bound to, its reason and the batch pull request that recorded it, so an +// earlier report still says why after the drop is withdrawn. +func TestReportKeepsEachDrop(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + mc6 := f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + stdout := mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "6", "--reason", "needs the rework of #4", "--push") + dropped := bodyReportOf(t, stdout) + want := `"overlays":{"dropped":[{"original":6,"merge":"` + mc6 + `","reason":"needs the rework of #4","on":90,"date":"2026-10-26"}]` + if !strings.Contains(dropped.raw, want) { + t.Errorf("the report does not keep the drop with its merge and reason:\n%s", dropped.raw) + } + mustRun(t, f, gh, "run", "--line", "release-1.1", "--undrop", "6", "--push") + if raw := bodyReportOf(t, gh.find(90).Body).raw; !strings.Contains(raw, `"overlays":{"dropped":[],`) { + t.Errorf("the report after the undrop still keeps a drop:\n%s", raw) + } + code, out, stderr := f.exec(gh, "report", "--pr", "90", "--id", dropped.doc.ID) + if code != 0 || !strings.Contains(out, "dropped: #6, merged as "+mc6[:12]+", on #90: needs the rework of #4\n") { + t.Errorf("report --id of the earlier report = %d, want it to show the drop:\n%s%s", code, out, stderr) + } +} + +// A report's drop is held to the record's rules and bound to main as the +// record's drops are: a reason a run does not write, another original's +// merge, or an earlier commit on main that names the same number is +// refused, however the ID was made to match, in the body and in a trusted +// report comment alike. +func TestReportRefusesAForgedDrop(t *testing.T) { + f := newFixture(t) + f.release("release-1.0") + f.tag("v1.0.0") + early := f.squash(6, "docs: what six will change", "", map[string]string{"docs/six.md": "6\n"}) + f.release("release-1.1") + f.tag("v1.1.0") + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + mc6 := f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "6", "--reason", "not for 1.1", "--push") + body, comments := gh.find(90).Body, slices.Clone(gh.issueComments) + s := bodyReportOf(t, body) + for _, tc := range []struct{ name, old, new, why string }{ + {"a reason of two lines", `"reason":"not for 1.1"`, `"reason":"not for 1.1\nwarning: forged"`, "which is no reason a run writes"}, + {"another original's merge", `"merge":"` + mc6 + `"`, `"merge":"` + mc5 + `"`, "which merged #5"}, + {"an earlier commit that names the same number", `"merge":"` + mc6 + `"`, `"merge":"` + early + `"`, "but #6 merged as " + mc6[:12]}, + } { + raw := strings.Replace(s.raw, tc.old, tc.new, 1) + if raw == s.raw { + t.Fatalf("the report holds no %s to change:\n%s", tc.old, s.raw) + } + var d reportDoc + if err := decodeStrict(raw, &d); err != nil { + t.Fatalf("the changed report does not decode: %v", err) + } + d.ID = reportID(d) + forged := storedReport{doc: d, raw: reportJSON(d)} + refused := func(t *testing.T, args ...string) { + t.Helper() + code, out, stderr := f.exec(gh, append([]string{"report", "--pr", "90"}, args...)...) + if code != 2 || !strings.Contains(stderr, tc.why) || strings.Contains(out+stderr, "\nwarning: forged") { + t.Errorf("report of a forged drop = %d, want 2 with %q:\n%s%s", code, tc.why, out, stderr) + } + } + t.Run(tc.name+", in the body", func(t *testing.T) { + gh.find(90).Body = strings.Replace(body, reportBlock(s.raw), reportBlock(forged.raw), 1) + defer func() { gh.find(90).Body = body }() + refused(t) + }) + t.Run(tc.name+", in a comment", func(t *testing.T) { + text, err := reportComment(forged) + if err != nil { + t.Fatal(err) + } + c := issueComment{ID: 9000, Body: text} + c.User.Login, c.User.Type = "maintainer", "User" + gh.issueComments = append(slices.Clone(comments), c) + defer func() { gh.issueComments = comments }() + refused(t, "--id", d.ID) + }) + } +} + +// droppedNumbers are the originals a report's drops name. +func droppedNumbers(es []droppedEntry) []int { + var out []int + for _, e := range es { + out = append(out, e.Original) + } + return out +} + +// dropWorld has #3 merged before release-1.1 branched, an earlier commit on +// main that names #6, and a batch with #5 and #6 from which #6 is dropped. +func dropWorld(t *testing.T) (f *fixture, gh *fakeGH, inherited, mc5, mc6 string) { + t.Helper() + f = newFixture(t) + f.release("release-1.0") + f.tag("v1.0.0") + inherited = f.prCommits(3, "feat(app): three", "", change{files: map[string]string{"app/three.txt": "3\n"}, message: "feat(app): part of three (#3)"}) + f.squash(6, "docs: what six will change", "", map[string]string{"docs/six.md": "6\n"}) + f.release("release-1.1") + f.tag("v1.1.0") + mc5 = f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + mc6 = f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh = &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "6", "--reason", "not for 1.1", "--push") + return f, gh, inherited, mc5, mc6 +} + +// A report's drops and its categories hold each other up: an original it +// has dropped needs a drop that binds to it, and a drop of an original it +// classified needs that original dropped or decided by an earlier rule. A +// drop inherited from before the line branched decides nothing and stays +// readable. Each holds in the body and in a trusted report comment. +func TestReportTiesDropsToTheirCategory(t *testing.T) { + f, gh, inherited, mc5, _ := dropWorld(t) + body, comments := gh.find(90).Body, slices.Clone(gh.issueComments) + s := bodyReportOf(t, body) + kept := func(n int, merge string) droppedEntry { + return droppedEntry{Original: n, Merge: merge, Reason: "carried from an earlier batch", On: 89, Date: "2026-10-25"} + } + for _, tc := range []struct { + name string + change func(d *reportDoc) + why string + }{ + {"an inherited drop in place of the classified one", func(d *reportDoc) { d.Overlays.Dropped[0] = kept(3, inherited) }, "has #6 dropped, but keeps no drop of it"}, + {"an inherited drop classified as dropped", func(d *reportDoc) { + d.Primary.Skipped.Dropped = append(d.Primary.Skipped.Dropped, 3) + d.Overlays.Dropped = append(d.Overlays.Dropped, kept(3, inherited)) + }, "has #3 dropped, but no drop of it binds to an original it classified"}, + {"a drop of an original it picked", func(d *reportDoc) { d.Overlays.Dropped = append(d.Overlays.Dropped, kept(5, mc5)) }, "drops #5, which it has picked"}, + {"an inherited drop beside the classified one", func(d *reportDoc) { d.Overlays.Dropped = append(d.Overlays.Dropped, kept(3, inherited)) }, ""}, + } { + var d reportDoc + if err := decodeStrict(s.raw, &d); err != nil { + t.Fatal(err) + } + tc.change(&d) + d.canonical() + d.ID = reportID(d) + forged := storedReport{doc: d, raw: reportJSON(d)} + check := func(t *testing.T, args ...string) { + t.Helper() + code, out, stderr := f.exec(gh, append([]string{"report", "--pr", "90"}, args...)...) + switch { + case tc.why == "" && (code != 0 || !strings.Contains(out, "#6 skipped.dropped") || !strings.Contains(out, "dropped: #3, merged as "+inherited[:12]) || strings.Contains(out, "#3 skipped.dropped")): + t.Errorf("report of a sound report = %d, want it read with #3's drop beside #6's:\n%s%s", code, out, stderr) + case tc.why != "" && (code != 2 || out != "" || !strings.Contains(stderr, tc.why)): + t.Errorf("report = %d, want 2 with %q:\n%s%s", code, tc.why, out, stderr) + } + } + t.Run(tc.name+", in the body", func(t *testing.T) { + gh.find(90).Body = strings.Replace(body, reportBlock(s.raw), reportBlock(forged.raw), 1) + defer func() { gh.find(90).Body = body }() + check(t) + }) + t.Run(tc.name+", in a comment", func(t *testing.T) { + text, err := reportComment(forged) + if err != nil { + t.Fatal(err) + } + c := issueComment{ID: 9001, Body: text} + c.User.Login, c.User.Type = "maintainer", "User" + gh.issueComments = append(slices.Clone(comments), c) + defer func() { gh.issueComments = comments }() + check(t, "--id", d.ID) + }) + } +} diff --git a/cmd/backport-batch/effects.go b/cmd/backport-batch/effects.go new file mode 100644 index 000000000..350f9a961 --- /dev/null +++ b/cmd/backport-batch/effects.go @@ -0,0 +1,625 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +// A clean pick can still carry only part of an original: git deletes a file +// the line never had without a word, and leaves alone a file the line lacks. +// What follows tells such a unit from a whole one, by what it leaves without +// effect of the files the original modifies or deletes. + +// treeEntry is one record of git ls-tree. +type treeEntry struct{ mode, kind, oid string } + +// regular says whether git can merge the entry as text: a file, not a +// symlink, a submodule or a tree. +func (e treeEntry) regular() bool { + return e.kind == "blob" && (e.mode == "100644" || e.mode == "100755") +} + +// treeEntries reads the entries paths have at treeish, from full ls-tree +// records: a path that is a tree there, or absent, has none. +func treeEntries(treeish string, paths []string) (map[string]treeEntry, error) { + out := map[string]treeEntry{} + want := map[string]bool{} + for _, p := range paths { + want[p] = true + } + const chunk = 500 + for len(paths) > 0 { + n := min(chunk, len(paths)) + args := append([]string{"--literal-pathspecs", "ls-tree", "-r", "-z", "--full-tree", "--end-of-options", treeish, "--"}, paths[:n]...) + raw, stderr, code, err := gitStatus(args...) + switch { + case err != nil: + return nil, fmt.Errorf("git ls-tree %s did not run", terminalSafe(treeish)) + case code != 0: + // The paths stay out of the error: they are file names someone + // chose, and the error reaches a terminal. + return nil, fmt.Errorf("git ls-tree %s: exit %d: %s", terminalSafe(treeish), code, terminalSafe(stderr)) + } + for _, rec := range splitNUL(raw) { + meta, path, ok := strings.Cut(rec, "\t") + f := strings.Fields(meta) + if !ok || len(f) != 3 || checkSHA(f[2]) != nil { + return nil, fmt.Errorf("git ls-tree %s: unreadable record %q", treeish, terminalSafe(rec)) + } + if want[path] { + out[path] = treeEntry{mode: f[0], kind: f[1], oid: f[2]} + } + } + paths = paths[n:] + } + return out, nil +} + +// renameSet is what git diff -M found renamed between two commits, old path +// to new, and whether it looked at every candidate. +type renameSet struct { + moved map[string]string + complete bool +} + +// renames reads the renames between from and to, once per pair. git says on +// stderr when it skipped inexact detection for too many files; complete is +// then false, and no path may be called absent for want of a rename. +func (b *batch) renames(from, to string) (renameSet, error) { + key := from + ".." + to + if s, ok := b.renameCache[key]; ok { + return s, nil + } + // -l0 lifts git's own limit, which the picker's merge does not share: + // a rename the pick follows has to be one the checks of its shape see. + stdout, stderr, code, err := gitStatusTimed("diff", "--no-ext-diff", "--no-textconv", "-M", "-l"+strconv.Itoa(b.renameLimit), "--name-status", "-z", "--diff-filter=R", "--end-of-options", from, to) + switch { + case errors.Is(err, errDeadline): + // Out of time is rename detection skipped: no path is called + // absent for want of it. + s := renameSet{moved: map[string]string{}} + if b.renameCache == nil { + b.renameCache = map[string]renameSet{} + } + b.renameCache[key] = s + return s, nil + case err != nil: + return renameSet{}, err + case code != 0: + return renameSet{}, fmt.Errorf("git diff -M %s %s: exit %d: %s", from, to, code, strings.TrimSpace(stderr)) + } + f := splitNUL(stdout) + if len(f)%3 != 0 { + return renameSet{}, fmt.Errorf("unreadable renames between %s and %s", from, to) + } + s := renameSet{moved: map[string]string{}, complete: !strings.Contains(stderr, "detection was skipped")} + for i := 0; i < len(f); i += 3 { + if !strings.HasPrefix(f[i], "R") { + return renameSet{}, fmt.Errorf("unreadable renames between %s and %s: %q", from, to, terminalSafe(f[i])) + } + s.moved[f[i+1]] = f[i+2] + } + if b.renameCache == nil { + b.renameCache = map[string]renameSet{} + } + b.renameCache[key] = s + return s, nil +} + +// effectPaths are the paths v modifies or deletes; what it adds has nothing +// to correspond to. +func effectPaths(v *verdict) []string { + var out []string + for _, c := range v.changes { + if c.status == 'M' || c.status == 'D' { + out = append(out, c.path) + } + } + return out +} + +// correspondence maps a path of the original (at M^1) to the path holding the +// same file on target, through a main-side rename base..M^1, a line-side +// rename base..target, or both composed through the base path. A path with +// no counterpart maps to itself, and one git skipped rename detection for is +// left out and named in unknown: it is not known to be absent. +func (b *batch) correspondence(v *verdict, target string) (map[string]string, []string, error) { + paths := effectPaths(v) + have, err := treeEntries(target, paths) + if err != nil { + return nil, nil, err + } + corr := map[string]string{} + var missing []string + for _, p := range paths { + if _, ok := have[p]; ok { + corr[p] = p + } else { + missing = append(missing, p) + } + } + if len(missing) == 0 { + return corr, nil, nil + } + onMain, err := b.renames(b.base, v.o.merge+"^1") + if err != nil { + return nil, nil, err + } + onLine, err := b.renames(b.base, target) + if err != nil { + return nil, nil, err + } + fromBase := map[string]string{} + for old, now := range onMain.moved { + fromBase[now] = old + } + candidates := map[string]string{} + var names []string + for _, p := range missing { + q := p + if old, ok := fromBase[p]; ok { + q = old + } + r := q + if now, ok := onLine.moved[q]; ok { + r = now + } + if r != p { + candidates[p] = r + names = append(names, r) + } + } + found, err := treeEntries(target, names) + if err != nil { + return nil, nil, err + } + var unknown []string + for _, p := range missing { + if r, ok := candidates[p]; ok { + if _, ok := found[r]; ok { + corr[p] = r + continue + } + } + if !onMain.complete || !onLine.complete { + unknown = append(unknown, p) + continue + } + corr[p] = p + } + return corr, unknown, nil +} + +// omittedEffects lists the paths v modifies or deletes that the change from +// before to after leaves without effect, each judged at its corresponding +// path c(p) on before. A modified path is omitted when c(p) exists on neither +// side, applied when the change touches it, and when the change leaves an +// existing file alone, satisfied only if the line already has what the +// original makes of it. A deleted path is omitted when c(p) is absent before: +// a deletion the line already did is no change this unit makes. diag says +// what could not be judged, which is never counted as omitted. +func (b *batch) omittedEffects(v *verdict, before, after string) ([]string, []string, error) { + if err := b.load(v); err != nil { + return nil, nil, err + } + corr, unknown, err := b.correspondence(v, before) + if err != nil { + return nil, nil, err + } + var diag []string + for _, p := range unknown { + diag = append(diag, "git skipped rename detection, so where "+strconv.Quote(p)+" is on the line is not known") + } + added := map[string]bool{} + for _, c := range v.changes { + if c.status == 'A' { + added[c.path] = true + } + } + // A deleted path whose counterpart the original adds itself is a rename + // the line already made the same way: it is judged as a modified file, + // against what the original makes of that path. + var at, own, moved []string + movedTo := map[string]string{} + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok { + continue + } + at = append(at, cp) + switch { + case c.status == 'M': + own = append(own, c.path) + case c.status == 'D' && cp != c.path && added[cp]: + own, moved = append(own, c.path), append(moved, cp) + movedTo[c.path] = cp + } + } + bef, err := treeEntries(before, at) + if err != nil { + return nil, nil, err + } + aft, err := treeEntries(after, at) + if err != nil { + return nil, nil, err + } + pre, err := treeEntries(v.o.merge+"^1", own) + if err != nil { + return nil, nil, err + } + post, err := treeEntries(v.o.merge, append(slices.Clone(own), moved...)) + if err != nil { + return nil, nil, err + } + // A change that makes none is judged by its trees, whatever names them: + // a commit and the tree a net merge wrote can be the same change. + same := before == after + if !same { + bt, err := resolveTree(before) + if err != nil { + return nil, nil, err + } + at, err := resolveTree(after) + if err != nil { + return nil, nil, err + } + same = bt == at + } + var omitted []string + // modified judges a path whose counterpart cp should end up as the + // original's postimage p1 of it. + modified := func(path, cp string, p0, p1 treeEntry) error { + be, inBefore := bef[cp] + ae, inAfter := aft[cp] + switch { + case !inBefore && !inAfter: + omitted = append(omitted, path) + return nil + case inBefore != inAfter || be != ae: + return nil + case be.oid == p1.oid && (p0.mode == p1.mode || be.mode == p1.mode): + return nil + case !be.regular() || !p0.regular() || !p1.regular() || p0.mode != p1.mode: + diag = append(diag, "the mode or type of "+strconv.Quote(path)+" changes, which only a person can judge") + return nil + } + applied, unsupported, err := b.alreadyApplied(p0.oid, be.oid, p1.oid) + switch { + case err != nil: + return fmt.Errorf("judging %s of #%d: %w", strconv.Quote(path), v.o.number, err) + case unsupported: + diag = append(diag, "git cannot merge "+strconv.Quote(path)+" as text, so whether the line has its change is not known") + case !applied: + omitted = append(omitted, path) + } + return nil + } + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok { + continue + } + switch { + case c.status == 'D' && movedTo[c.path] != "": + if err := modified(c.path, cp, pre[c.path], post[cp]); err != nil { + return nil, nil, err + } + case c.status == 'D': + // Absent before: the line already made the deletion, which is + // no change of a unit that makes others, and is all of one + // that makes none. Still there after: the deletion did not + // happen, whatever the unit changed in the file; keeping what + // the line made of it is a port of its own. + _, inBefore := bef[cp] + _, inAfter := aft[cp] + if !inBefore && !same || inAfter { + omitted = append(omitted, c.path) + } + case c.status == 'M': + if err := modified(c.path, cp, pre[c.path], post[c.path]); err != nil { + return nil, nil, err + } + } + } + for i := range diag { + diag[i] = oneLineText(diag[i]) + } + return omitted, diag, nil +} + +// alreadyApplied says whether cur, a file the unit leaves alone, already +// holds the original's change from pre to post: it is post itself, or git +// merge-file of the three merges cleanly back into cur. unsupported is +// content merge-file refuses, such as differing binary files, which says +// nothing either way. +func (b *batch) alreadyApplied(pre, cur, post string) (applied, unsupported bool, err error) { + if cur == post { + return true, false, nil + } + b.mergeFiles++ + blobs, err := readObjects([]string{pre, cur, post}, "blob") + if err != nil { + return false, false, err + } + dir, err := os.MkdirTemp("", "backport-batch-merge-file-*") + if err != nil { + return false, false, err + } + defer os.RemoveAll(dir) + names := map[string]string{} + for _, f := range []struct{ name, oid string }{{"cur", cur}, {"pre", pre}, {"post", post}} { + names[f.name] = filepath.Join(dir, f.name) + if err := os.WriteFile(names[f.name], blobs[f.oid], 0o600); err != nil { + return false, false, err + } + } + stdout, stderr, code, err := gitStatus("merge-file", "-p", names["cur"], names["pre"], names["post"]) + switch { + case err != nil: + return false, false, err + case code == 0: + return stdout == string(blobs[cur]), false, nil + case code > 0 && code < 128: + // The number of conflicts. + return false, false, nil + case strings.Contains(stderr, "Cannot merge binary files"): + return false, true, nil + } + return false, false, fmt.Errorf("git merge-file: exit %d: %s", code, strings.TrimSpace(stderr)) +} + +// unitsOn maps each original the batch head carries a unit of to that unit, +// by its Backported-from trailer or appended -x line: found needs every +// commit of a merged original, and misses one whose first pick came out +// empty because the line already had that change. +func (b *batch) unitsOn(head string) (map[*verdict]string, error) { + if m, ok := b.unitCache[head]; ok { + return m, nil + } + units, err := batchUnits(head, b.lineHead) + if err != nil { + return nil, err + } + m := map[*verdict]string{} + for _, u := range units { + if v := b.unitOriginal(u); v != nil && m[v] == "" { + m[v] = u.commit + } + } + if b.unitCache == nil { + b.unitCache = map[string]map[*verdict]string{} + } + b.unitCache[head] = m + return m, nil +} + +// onBranch says whether the batch head h carries v. +func (b *batch) onBranch(h *history, v *verdict) (bool, error) { + if found(h, v) { + return true, nil + } + m, err := b.unitsOn(h.head) + return m[v] != "", err +} + +// partialStates finds what applies only in part on the line. Every unit on +// the open batch is checked against its own first parent, whatever the rules +// now say of its original: one that leaves an effect out fails the cut until +// it is taken out. Every original the policy takes and nothing carries is +// checked against the tree its clean net merge onto the line writes, so a +// partial one is found with no record and no adder; a conflict is never +// called partial. +func (b *batch) partialStates() error { + defer b.timed(time.Now()) + b.partialUnits = map[*verdict][]string{} + if b.pr != nil { + var err error + if b.partialUnits, err = b.unitsPartial(b.prHead); err != nil { + return err + } + on, err := b.unitsOn(b.prHead) + if err != nil { + return err + } + for v, commit := range on { + v.batched = commit + if v.rule == 0 && v.onBatch == "" { + v.onBatch = commit + } + } + } + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + if err := b.partialAt(v, b.lineHead); err != nil { + return err + } + } + return nil +} + +// unitsPartial maps each original whose unit between the line's head and +// head leaves part of it without effect to what it leaves. +func (b *batch) unitsPartial(head string) (map[*verdict][]string, error) { + units, err := batchUnits(head, b.lineHead) + if err != nil { + return nil, err + } + out := map[*verdict][]string{} + for _, u := range units { + v := b.unitOriginal(u) + if v == nil || len(u.parents) == 0 { + continue + } + omitted, diag, err := b.omittedEffects(v, u.parents[0], u.commit) + if err != nil { + return nil, err + } + v.addDiag(diag...) + if len(omitted) > 0 { + out[v] = append(out[v], omitted...) + } + } + return out, nil +} + +// markPartial puts every original on the branch at head whose unit there +// does only part of it among the partial units, and so among the stale +// ones: still on the batch, and to be taken out before the cut. +func (b *batch) markPartial(r *report, head string) error { + units := b.partialUnits + if b.pr == nil || head != b.prHead { + var err error + if units, err = b.unitsPartial(head); err != nil { + return err + } + } + r.partialFiles = map[*verdict][]string{} + for _, v := range b.verdicts { + files := units[v] + if len(files) == 0 { + continue + } + r.partial = append(r.partial, v) + r.partialFiles[v] = files + if !slices.Contains(r.stale, v) { + r.stale = append(r.stale, v) + } + } + return nil +} + +// keepRecordedPartials judges again, at head, what the batch carries now, +// each original the open record parks as partial that the line's head +// judges whole: the run's picker found it partial after the picks before +// it, which check and an operation that picks nothing do not replay. It +// stays parked while its net merge onto head still leaves something +// without effect, and is the line's again once that went, as when the pick +// that made it partial was taken off. +func (b *batch) keepRecordedPartials(head string) error { + if !b.src.open || b.src.rec == nil { + return nil + } + for _, p := range b.src.rec.Parked { + v := b.find(p.Original) + if p.Kind != parkPartial || v == nil || v.rule != 0 || v.onBatch != "" || v.park != nil || v.o.merge != p.Merge { + continue + } + if err := b.partialAt(v, head); err != nil { + return err + } + } + return nil +} + +// rejudged reads again what partialStates and analyze read for v once its +// rules changed, as a withdrawn drop changes them: it is back where the +// line puts it. +func (b *batch) rejudged(v *verdict) error { + if b.pr != nil { + on, err := b.unitsOn(b.prHead) + if err != nil { + return err + } + v.batched = on[v] + if v.rule == 0 && v.onBatch == "" { + v.onBatch = on[v] + } + } + if v.rule != 0 || v.onBatch != "" { + return nil + } + if err := b.partialAt(v, b.lineHead); err != nil { + return err + } + deps, diag, err := b.dependencies(v, b.lineHead) + if err != nil { + return err + } + v.deps = deps + v.addDiag(diag...) + return nil +} + +// partialAt judges v's clean net merge onto head, and parks it, or clears +// the park, by what that merge leaves without effect. +func (b *batch) partialAt(v *verdict, head string) error { + tree, err := b.netTree(head, v.o) + if err != nil { + return err + } + v.partial = nil + if v.park != nil && v.park.Kind == parkPartial { + v.park = nil + } + if tree == "" { + return nil + } + omitted, diag, err := b.omittedEffects(v, head, tree) + if err != nil { + return err + } + v.addDiag(diag...) + if len(omitted) > 0 { + v.partial = omitted + v.park = &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkPartial, Files: sortedClean(omitted)} + } + return nil +} + +// netTree is the tree a clean net merge of o onto head writes, "" for a +// conflict; the line's head reuses what judging the originals merged. +func (b *batch) netTree(head string, o *original) (string, error) { + if head == b.lineHead && b.lineHist != nil { + return b.lineHist.netTree(o) + } + tree, _, err := noopPick(head, o) + return tree, err +} + +// addDiag keeps what could not be judged of v, each line once. +func (v *verdict) addDiag(lines ...string) { + for _, ln := range lines { + if ln != "" && !slices.Contains(v.diag, ln) { + v.diag = append(v.diag, ln) + } + } +} + +func sortedClean(files []string) []string { + out := cleanFiles(files) + slices.Sort(out) + return slices.Compact(out) +} + +// leftOut is the refusal for a unit that leaves paths of v without effect. +func leftOut(v *verdict, l line, omitted []string) string { + quoted := make([]string, len(omitted)) + for i, p := range omitted { + quoted[i] = strconv.Quote(p) + } + return fmt.Sprintf("#%d leaves %s without effect on %s; port only what the line has as a pull request of its own", v.o.number, strings.Join(quoted, ", "), l.branch()) +} diff --git a/cmd/backport-batch/effects_test.go b/cmd/backport-batch/effects_test.go new file mode 100644 index 000000000..f96f5a596 --- /dev/null +++ b/cmd/backport-batch/effects_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const appFile = "one\ntwo\nthree\nfour\nfive\n" + +// renameWorld is the sign-off's rename history: the line keeps app/old.txt, +// main renames it to app/new.txt with #3, a feat, and #5 fixes app/new.txt. +func renameWorld(t *testing.T) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: an app file", "", map[string]string{"app/old.txt": appFile}) + f.lines() + f.pr(3, "feat(app): rename the app file", "", map[string]string{"app/old.txt": "", "app/new.txt": appFile}) + mc5 := f.pr(5, "fix(app): fix the app file", "", map[string]string{"app/new.txt": strings.Replace(appFile, "three", "THREE", 1)}) + return f, mc5 +} + +// emptyBatch pushes the line's head as the batch branch and opens a batch +// pull request on it with body, for a fold to land on. +func emptyBatch(f *fixture, gh *fakeGH, body string) *foldWorld { + f.t.Helper() + f.git("fetch", "-q", "origin") + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+batchBranch1) + f.git("fetch", "-q", "origin") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, body)} + return &foldWorld{f: f, gh: gh, head: f.git("rev-parse", "origin/"+batchBranch1)} +} + +// A fix of a file main renamed picks onto the file's old name on the line, +// and every check of the batch's shape takes the old name as the same file: +// verify, a hand fold, and a rebuild by --defer. +func TestRenamePickThroughFoldAndVerify(t *testing.T) { + f, mc5 := renameWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): fix the app file (#5)", "fix(app): six (#6)"}) { + t.Fatalf("the batch carries %q, want #5 and #6", got) + } + if got := f.git("show", head+":app/old.txt"); !strings.Contains(got, "THREE") { + t.Fatalf("#5 did not land on app/old.txt: %q", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + t.Setenv("GIT_COMMITTER_DATE", "2026-10-27T06:00:00Z") + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e", "--push"); code != 0 { + t.Fatalf("run --defer 6 --push = %d\n%s", code, stderr) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", f.remoteHead(batchBranch1)); code != 0 { + t.Errorf("verify after the rebuild = %d\n%s\n%s", code, out, stderr) + } + + f, mc5 = renameWorld(t) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/old.txt": strings.Replace(appFile, "three", "THREE", 1)}) + fold := w.merge(w.head, pick, "fix(app): fix the app file (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 || out != "ok\n" { + t.Errorf("fold of #5 onto app/old.txt = %d, %q\n%s", code, out, stderr) + } +} + +// shaped reads the line from the fixture's remote as verify does. +func shaped(t *testing.T, f *fixture) *batch { + t.Helper() + f.git("fetch", "-q", "origin") + b, err := shapeOf(line11) + if err != nil { + t.Fatal(err) + } + return b +} + +// commitOn writes files onto parent and returns the commit, leaving every +// branch where it was. +func commitOn(f *fixture, parent string, files map[string]string) string { + f.t.Helper() + f.git("switch", "-q", "--detach", parent) + f.apply(files) + f.git("commit", "-q", "--allow-empty", "-m", "a unit") + sha := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + return sha +} + +// A path of the original is followed to where the line keeps the same file: +// through a rename on main, on the line, or both, composed through the path +// at the base. Where git skipped rename detection the path is unknown, never +// absent. +func TestCorrespondence(t *testing.T) { + corr := func(t *testing.T, f *fixture, n int, limit ...int) (map[string]string, []string) { + t.Helper() + b := shaped(t, f) + if len(limit) > 0 { + b.renameLimit = limit[0] + } + v := b.find(n) + if err := b.load(v); err != nil { + t.Fatal(err) + } + c, unknown, err := b.correspondence(v, b.lineHead) + if err != nil { + t.Fatal(err) + } + return c, unknown + } + t.Run("renamed on main", func(t *testing.T) { + f, _ := renameWorld(t) + if c, unknown := corr(t, f, 5); c["app/new.txt"] != "app/old.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/new.txt on app/old.txt", c, unknown) + } + }) + t.Run("renamed on the line", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: an app file", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames it", map[string]string{"app/a.txt": "", "app/line.txt": appFile}) + f.pr(5, "fix(app): fix it", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + if c, unknown := corr(t, f, 5); c["app/a.txt"] != "app/line.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/a.txt on app/line.txt", c, unknown) + } + }) + t.Run("renamed on both", func(t *testing.T) { + f, _ := renameWorld(t) + f.onLine("release-1.1", "fix(app): the line renames it its own way", map[string]string{"app/old.txt": "", "app/release.txt": appFile}) + if c, unknown := corr(t, f, 5); c["app/new.txt"] != "app/release.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/new.txt on app/release.txt", c, unknown) + } + }) + t.Run("not renamed, absent", func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix x", "", map[string]string{"app/x.txt": strings.Replace(appFile, "two", "TWO", 1)}) + if c, unknown := corr(t, f, 5); c["app/x.txt"] != "app/x.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/x.txt on itself", c, unknown) + } + }) + t.Run("rename detection skipped", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: two app files", "", map[string]string{"app/a.txt": appFile, "app/b.txt": appFile + "six\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames both, changed", map[string]string{"app/a.txt": "", "app/b.txt": "", "app/x.txt": appFile + "x\n", "app/y.txt": appFile + "six\ny\n"}) + f.pr(5, "fix(app): fix a", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + // Past the limit, git says it skipped looking; a configured limit + // of its own does not decide for the tool. + f.git("config", "diff.renameLimit", "1") + if c, unknown := corr(t, f, 5, 0); c["app/a.txt"] != "app/x.txt" || len(unknown) != 0 { + t.Errorf("correspondence under diff.renameLimit=1 = %v, %v; want app/a.txt on app/x.txt", c, unknown) + } + if c, unknown := corr(t, f, 5, 1); !slices.Equal(unknown, []string{"app/a.txt"}) || c["app/a.txt"] != "" { + t.Errorf("correspondence past the limit = %v, %v; want app/a.txt unknown", c, unknown) + } + }) +} + +// What a change leaves without effect: a modified file the line lacks on both +// sides of it, a deleted file the line never had. A mode change that only a +// person can judge is a diagnostic, not an omission. +func TestOmittedEffects(t *testing.T) { + omitted := func(t *testing.T, f *fixture, n int, files map[string]string) ([]string, []string) { + t.Helper() + b := shaped(t, f) + after := commitOn(f, b.lineHead, files) + om, diag, err := b.omittedEffects(b.find(n), b.lineHead, after) + if err != nil { + t.Fatal(err) + } + return om, diag + } + t.Run("a modified file missing before and after", func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix x and five", "", map[string]string{"app/x.txt": "fixed\n", "app/five.txt": "5\n"}) + if om, diag := omitted(t, f, 5, map[string]string{"app/five.txt": "5\n"}); !slices.Equal(om, []string{"app/x.txt"}) || len(diag) != 0 { + t.Errorf("omitted = %q, %q; want app/x.txt", om, diag) + } + }) + t.Run("a deleted file absent before", func(t *testing.T) { + f, _ := policyWorld(t) + if om, _ := omitted(t, f, 5, map[string]string{"chart/tests/values_test.yaml": "suite: values\n"}); !slices.Equal(om, []string{"chart/templates/policy.yaml", "chart/tests/policy_test.yaml"}) { + t.Errorf("omitted = %q; want both policy files", om) + } + }) + t.Run("a mode change", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a script", "", map[string]string{"app/run.sh": "#!/bin/sh\n", "app/b.txt": "old\n"}) + f.lines() + f.git("switch", "-q", "-c", "pr-5") + if err := os.Chmod(filepath.Join(f.dir, "app/run.sh"), 0o755); err != nil { + t.Fatal(err) + } + f.write("app/b.txt", "fixed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "run it") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): make the script runnable (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + om, diag := omitted(t, f, 5, map[string]string{"app/b.txt": "fixed\n"}) + if len(om) != 0 || len(diag) != 1 || !strings.Contains(diag[0], `the mode or type of "app/run.sh" changes`) { + t.Errorf("omitted = %q, %q; want a diagnostic for app/run.sh and nothing omitted", om, diag) + } + }) +} + +// What git says when it fails names file names as text, never as lines a +// terminal or a CI log would take for its own. +func TestGitErrorsQuoteFileNames(t *testing.T) { + newFixture(t) + const evil = "x\n::warning::forged" + if _, err := gitTimed("log", "--no-such-option", "--", evil); err == nil || strings.Contains(err.Error(), "\n::warning") { + t.Errorf("gitTimed error = %v, want the path quoted", err) + } + if _, err := treeEntries("no-such-revision", []string{evil}); err == nil || strings.Contains(err.Error(), "\n::warning") { + t.Errorf("treeEntries error = %v, want the path quoted", err) + } +} + +// fixedWorld is the sign-off's two-file case: the base has a and b as old, +// #5 fixes both, and the line already has a as onLine says, if anything. +func fixedWorld(t *testing.T, a, old, onLine, fixed string) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{a: old, "app/b.txt": "old\n"}) + f.lines() + if onLine != "" { + f.onLine("release-1.1", "fix(app): a, on the line", map[string]string{a: onLine}) + } + mc5 := f.pr(5, "fix(app): a and b", "", map[string]string{a: fixed, "app/b.txt": "fixed\n"}) + return f, mc5 +} + +// The line can already have what an original does to a file: then a pick +// that leaves the file alone carries the original whole. The check-time +// pass, the picker, fold and verify all take it, binary files included, +// without asking git merge-file what blob equality settles. A different fix +// on the line, or a deletion of what the line never had, is still left out. +func TestAlreadyAppliedFileIsNotOmitted(t *testing.T) { + const textPost = "one\nfixed\nthree\nfour\nfive\n" + accepted := func(t *testing.T, a, old, onLine, fixed string) { + t.Helper() + f, _ := fixedWorld(t, a, old, onLine, fixed) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check = %d, want #5 missing and not parked:\n%s\n%s", code, stdout, stderr) + } + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a and b (#5)"}) { + t.Errorf("picked %q, want #5", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || strings.Contains(stdout, "only part of it applies") { + t.Errorf("check --cut = %d, want #5 only on the batch, whole:\n%s\n%s", code, stdout, stderr) + } + f, mc5 := fixedWorld(t, a, old, onLine, fixed) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/b.txt": "fixed\n"}) + fold := w.merge(w.head, pick, "fix(app): a and b (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 { + t.Errorf("fold = %d, %q\n%s", code, out, stderr) + } + } + t.Run("the line has the original's file", func(t *testing.T) { + accepted(t, "app/a.txt", "old\n", "fixed\n", "fixed\n") + }) + t.Run("the line has the fix beside lines of its own", func(t *testing.T) { + accepted(t, "app/a.txt", strings.Replace(textPost, "fixed", "old", 1), textPost+"line only\n", textPost) + }) + t.Run("the line has the original's binary file", func(t *testing.T) { + accepted(t, "app/a.bin", "old\x00payload", "fixed\x00payload", "fixed\x00payload") + f, _ := fixedWorld(t, "app/a.bin", "old\x00payload", "fixed\x00payload", "fixed\x00payload") + b := shaped(t, f) + om, diag, err := b.omittedEffects(b.find(5), b.lineHead, commitOn(f, b.lineHead, map[string]string{"app/b.txt": "fixed\n"})) + if err != nil || len(om)+len(diag) != 0 || b.mergeFiles != 0 { + t.Errorf("omitted = %q, %q, %v, merge-file ran %d times; want nothing, and merge-file never asked", om, diag, err, b.mergeFiles) + } + }) + t.Run("an earlier unit of the batch brought the file", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old\n", "app/b.txt": "old\n"}) + f.lines() + f.pr(4, "fix(app): a", "", map[string]string{"app/a.txt": "fixed\n"}) + f.squash(40, "chore: put a back for now", "", map[string]string{"app/a.txt": "old\n"}) + f.pr(5, "fix(app): a and b", "", map[string]string{"app/a.txt": "fixed\n", "app/b.txt": "fixed\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a (#4)", "fix(app): a and b (#5)"}) { + t.Errorf("picked %q, want #4 and #5", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + }) + t.Run("the line has a different fix", func(t *testing.T) { + f, mc5 := fixedWorld(t, "app/a.txt", "old\n", "other\n", "fixed\n") + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/b.txt": "fixed\n"}) + fold := w.merge(w.head, pick, "fix(app): a and b (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/a.txt" without effect`) { + t.Errorf("fold = %d, %q, want it refused for app/a.txt\n%s", code, out, stderr) + } + }) + t.Run("the line has a different binary file", func(t *testing.T) { + f, _ := fixedWorld(t, "app/a.bin", "old\x00payload", "other\x00payload", "fixed\x00payload") + b := shaped(t, f) + om, diag, err := b.omittedEffects(b.find(5), b.lineHead, commitOn(f, b.lineHead, map[string]string{"app/b.txt": "fixed\n"})) + if err != nil || len(om) != 0 || len(diag) != 1 || !strings.Contains(diag[0], `cannot merge "app/a.bin" as text`) { + t.Errorf("omitted = %q, %q, %v; want a diagnostic for app/a.bin, nothing omitted", om, diag, err) + } + }) + t.Run("a deletion of what the line never had", func(t *testing.T) { + f, mc5 := policyWorld(t) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"chart/tests/values_test.yaml": "suite: values\n"}) + fold := w.merge(w.head, pick, "fix(chart): remove the policy a rebase brought back (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, "without effect") { + t.Errorf("fold = %d, %q, want it refused\n%s", code, out, stderr) + } + }) +} + +// A change that makes none, whatever names its two sides, leaves a deletion +// the line already made as it is: the original is there, not partial. +func TestDeletionAlreadyMadeByNoChange(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(5, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": ""}) + b := shaped(t, f) + tree := f.git("rev-parse", b.lineHead+"^{tree}") + for _, after := range []string{b.lineHead, tree} { + if om, _, err := b.omittedEffects(b.find(5), b.lineHead, after); err != nil || len(om) != 0 { + t.Errorf("omitted against %.12s = %q, %v; want nothing", after, om, err) + } + } +} diff --git a/cmd/backport-batch/fixture_test.go b/cmd/backport-batch/fixture_test.go index cb14f8e7e..ac1307c46 100644 --- a/cmd/backport-batch/fixture_test.go +++ b/cmd/backport-batch/fixture_test.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "cmp" "fmt" "os" "os/exec" @@ -32,13 +33,15 @@ import ( // main and release lines shaped the way GitHub leaves them: every pull request // lands as a merge commit whose message is its title, its number and its body. type fixture struct { - t *testing.T + t testing.TB dir string remote string now time.Time + // tool is the commit the tool says it was built from. + tool string } -func newFixture(t *testing.T) *fixture { +func newFixture(t testing.TB) *fixture { t.Helper() // A hook or an outer repository can export these into the test process. for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} { @@ -251,15 +254,114 @@ type fakeGH struct { issueComments []issueComment // bot is what BACKPORT_BATCH_BOT would say. bot string + // listed counts the pages of recentPulls read. + listed int + // closedWrites names every write the tool attempted on a pull request + // that is no longer open; exec fails the test on one. + closedWrites []string + // fail makes the next call of an operation, by its method name, fail. + fail map[string]error + // nextPR is the number createPull gives, 90 when unset. + nextPR int + // remoteHead reads a branch's head off the fixture's remote: GitHub + // reports an open pull request at what its branch has. + remoteHead func(string) string + // commented lists the pull request each comment went to, in order. + commented []int + // after runs once an operation, by its method name, has succeeded: what + // someone else does on GitHub meanwhile. + after map[string]func() +} + +// live is pr as GitHub reports it now: an open one at the head its branch +// has on the remote. +func (g *fakeGH) live(pr pullRequest) pullRequest { + if g.remoteHead != nil && (pr.State == "" || pr.State == "open") { + if head := g.remoteHead(pr.Head.Ref); head != "" { + pr.Head.SHA = head + } + } + return pr +} + +// inject is the failure a test set for the next call of op, once. +func (g *fakeGH) inject(op string) error { + err := g.fail[op] + delete(g.fail, op) + return err +} + +func (g *fakeGH) pull(n int) (pullRequest, error) { + if err := g.inject("pull"); err != nil { + return pullRequest{}, err + } + for _, pr := range g.prs { + if pr.Number == n { + return g.live(pr), nil + } + } + return pullRequest{}, fmt.Errorf("HTTP 404: no pull request #%d", n) +} + +// isOpen says whether the fake has pull request n open, or does not know it. +func (g *fakeGH) isOpen(n int) bool { + for _, pr := range g.prs { + if pr.Number == n { + return pr.State == "" || pr.State == "open" + } + } + return true +} + +// write refuses a write to a pull request that is no longer open, which the +// tool must never attempt. +func (g *fakeGH) write(n int, what string) error { + if g.isOpen(n) { + return nil + } + g.closedWrites = append(g.closedWrites, fmt.Sprintf("%s on #%d", what, n)) + return fmt.Errorf("#%d is closed", n) +} + +// recentPulls pages every pull request against base, newest first, as the +// REST listing sorted by creation does; a higher number is a newer one. +func (g *fakeGH) recentPulls(base string, page int) ([]pullRequest, error) { + g.listed++ + var all []pullRequest + for _, pr := range g.prs { + if pr.Base.Ref == base { + all = append(all, pr) + } + } + slices.SortFunc(all, func(a, b pullRequest) int { return b.Number - a.Number }) + from := min(len(all), (page-1)*recentPullsPage) + return slices.Clone(all[from:min(len(all), from+recentPullsPage)]), g.err } func (g *fakeGH) batchBot() string { return g.bot } func (g *fakeGH) comments(int) ([]issueComment, error) { + defer func() { + if f := g.after["comments"]; f != nil { + f() + } + }() return slices.Clone(g.issueComments), nil } func (g *fakeGH) createComment(number int, bodyFile string) error { + if err := g.write(number, "a comment"); err != nil { + return err + } + if err := g.inject("createComment"); err != nil { + return err + } + g.commented = append(g.commented, number) + defer func() { + if f := g.after["createComment"]; f != nil { + f() + } + }() data, err := os.ReadFile(bodyFile) var c issueComment c.ID, c.Body = int64(1000+len(g.issueComments)), string(data) @@ -302,21 +404,99 @@ func (g *fakeGH) permission(login string) (string, error) { func (g *fakeGH) openPulls(base string) ([]pullRequest, error) { var out []pullRequest for _, pr := range g.prs { - if pr.Base.Ref == base { - out = append(out, pr) + if pr.Base.Ref == base && (pr.State == "" || pr.State == "open") { + out = append(out, g.live(pr)) } } return out, g.err } -func (g *fakeGH) createPull(base, head, title, bodyFile string) error { +// createPull opens the pull request on the fake, as the maintainer, at the +// head the remote has, numbered nextPR. +func (g *fakeGH) createPull(base, head, title, bodyFile string) (int, string, error) { + if err := g.inject("createPull"); err != nil { + return 0, "", err + } g.created = append(g.created, base+" "+head+" "+title) - return g.readBody(bodyFile) + if err := g.readBody(bodyFile); err != nil { + return 0, "", err + } + n := cmp.Or(g.nextPR, 90) + var pr pullRequest + pr.Number, pr.Body, pr.State = n, g.bodies[len(g.bodies)-1], "open" + pr.HTMLURL = fmt.Sprintf("https://github.com/%s/pull/%d", upstreamRepo, n) + pr.User.Login, pr.User.Type = "maintainer", "User" + pr.Head.Ref = head + if g.remoteHead != nil { + pr.Head.SHA = g.remoteHead(head) + } + pr.Head.Repo = &struct { + FullName string `json:"full_name"` + }{upstreamRepo} + pr.Base.Ref, pr.Base.Repo.FullName = base, upstreamRepo + g.prs = append(slices.DeleteFunc(g.prs, func(p pullRequest) bool { return p.Number == n }), pr) + return n, pr.HTMLURL, nil } func (g *fakeGH) editPull(number int, title, bodyFile string) error { + if err := g.write(number, "an edit"); err != nil { + return err + } + if err := g.inject("editPull"); err != nil { + return err + } g.edited = append(g.edited, fmt.Sprintf("#%d %s", number, title)) - return g.readBody(bodyFile) + if err := g.readBody(bodyFile); err != nil { + return err + } + if pr := g.find(number); pr != nil { + pr.Body = g.bodies[len(g.bodies)-1] + } + if f := g.after["editPull"]; f != nil { + f() + } + return nil +} + +func (g *fakeGH) editBody(number int, bodyFile string) error { + if err := g.write(number, "a body edit"); err != nil { + return err + } + if err := g.inject("editBody"); err != nil { + return err + } + if err := g.readBody(bodyFile); err != nil { + return err + } + if pr := g.find(number); pr != nil { + pr.Body = g.bodies[len(g.bodies)-1] + } + return nil +} + +func (g *fakeGH) closePull(number int) error { + if err := g.write(number, "a close"); err != nil { + return err + } + if err := g.inject("closePull"); err != nil { + return err + } + if pr := g.find(number); pr != nil { + pr.State = "closed" + } + if f := g.after["closePull"]; f != nil { + f() + } + return nil +} + +func (g *fakeGH) find(n int) *pullRequest { + for i := range g.prs { + if g.prs[i].Number == n { + return &g.prs[i] + } + } + return nil } func (g *fakeGH) readBody(path string) error { @@ -338,18 +518,55 @@ func (f *fixture) batchPR(n int, base, branch, body string) pullRequest { FullName string `json:"full_name"` }{upstreamRepo} pr.Base.Ref, pr.Base.Repo.FullName = base, upstreamRepo + pr.State = "open" return pr } -// exec runs the tool in the fixture with gh faked, on the fixture's date. +// botBatchPR is batchPR opened by the bot the scheduled batch runs as. +func (f *fixture) botBatchPR(gh *fakeGH, n int, base, branch, body string) pullRequest { + f.t.Helper() + gh.bot = defaultBatchBot + pr := f.batchPR(n, base, branch, body) + pr.User.Login, pr.User.Type, pr.AuthorAssociation = defaultBatchBot, "Bot", "CONTRIBUTOR" + return pr +} + +// mergeBatch merges the open batch into release-1.1 with a merge commit, as +// GitHub does, and deletes its branch, as the repository's +// delete_branch_on_merge does. The pull request stays, merged, with its body +// and comments. +func mergeBatch(f *fixture, gh *fakeGH) { + f.t.Helper() + f.git("fetch", "-q", "origin") + f.git("switch", "-q", "--detach", "origin/release-1.1") + f.git("merge", "-q", "--no-ff", "-m", "Merge pull request #90 from cozystack/"+batchBranch1, "origin/"+batchBranch1) + f.git("push", "-q", "origin", "HEAD:refs/heads/release-1.1", ":refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + for i := range gh.prs { + if gh.prs[i].Number == 90 { + at := "2026-10-26T12:00:00Z" + gh.prs[i].State, gh.prs[i].MergedAt = "closed", &at + } + } +} + +// exec runs the tool in the fixture with gh faked, on the fixture's date, and +// fails the test when the tool wrote to a pull request that is not open. func (f *fixture) exec(gh *fakeGH, args ...string) (int, string, string) { f.t.Helper() if gh == nil { gh = &fakeGH{} } + if gh.remoteHead == nil { + gh.remoteHead = f.remoteHead + } var stdout, stderr bytes.Buffer - a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return f.now }} + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return f.now }, tool: f.tool} + written := len(gh.closedWrites) code := a.main(args) + if len(gh.closedWrites) > written { + f.t.Errorf("%v wrote to a pull request that is not open: %q", args, gh.closedWrites[written:]) + } return code, stdout.String(), stderr.String() } diff --git a/cmd/backport-batch/fold_test.go b/cmd/backport-batch/fold_test.go index 87f46f681..f783802b2 100644 --- a/cmd/backport-batch/fold_test.go +++ b/cmd/backport-batch/fold_test.go @@ -330,3 +330,93 @@ func TestFold(t *testing.T) { } }) } + +// A resolution of an original a maintainer dropped from the line is refused: +// the drop is the decision that it does not go there. +func TestFoldRefusesADroppedOriginal(t *testing.T) { + w := newFoldWorld(t) + sha := strings.Repeat("a", 40) + rec := strings.Replace(v2(""), `"dropped":[{"original":8,"merge":"`+sha+`"`, `"dropped":[{"original":5,"merge":"`+w.mc5+`"`, 1) + rec = strings.Replace(rec, `"deferred":[{"original":5,"commit":"`+sha+`","files":["app/five.txt"]},`, `"deferred":[`, 1) + rec = strings.Replace(rec, `"stale":[9]`, `"stale":[]`, 1) + w.gh.prs[0].Body = "text\n\n" + deferredFence + "\n" + rec + "\n```\n" + if _, err := parseRecord(w.gh.prs[0].Body, notifyLine); err != nil { + t.Fatal(err) + } + code, stdout, stderr := w.f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", w.resolution()) + if code != 1 || !strings.Contains(stdout, "#5 is not one the batch takes: a maintainer's drop, dropped on #90: needs #4") { + t.Errorf("fold = %d, %q, %s; want a refusal naming the drop", code, stdout, stderr) + } +} + +// A resolution that ports what the line has of #5 and leaves alone a file +// the line lacks is no fold: what #5 does to that file would be lost, and a +// line-only port is a pull request of its own. +func TestFoldRefusesAnOmittedEffect(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): a feature the line never gets", "", map[string]string{"app/feature2.txt": "v1\n"}) + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/feature2.txt": "v2\n", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): five (#5)\n\n"+batchTrailer+": "+mc5) + code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold) + if code != 1 || out != "refused: #5 leaves \"app/feature2.txt\" without effect on release-1.1; port only what the line has as a pull request of its own\n" { + t.Errorf("fold = %d, %q, want it refused for app/feature2.txt\n%s", code, out, stderr) + } +} + +// A resolution that keeps a file the original deletes leaves the deletion +// without effect. +func TestFoldRefusesAKeptDeletion(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: x", "", map[string]string{"app/x.txt": "x\n"}) + f.lines() + mc5 := f.pr(5, "fix(app): five, and x goes", "", map[string]string{"app/x.txt": "", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): five, and x goes (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/x.txt" without effect`) { + t.Errorf("fold that keeps app/x.txt = %d, %q\n%s", code, out, stderr) + } +} + +// An original parked for what it builds on can still land as a complete +// hand adaptation: every file it changes, changed, in the batch's shape. +func TestFoldAcceptsACompleteAdaptationOfAParkedOriginal(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + mc5 := f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/a.txt": strings.Replace(appFile, "two", "two!", 1)}) + fold := w.merge(w.head, pick, "fix(app): an excited two (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 || out != "ok\n" { + t.Errorf("fold of an adaptation of #5 = %d, %q\n%s", code, out, stderr) + } +} + +// A fold that keeps a file the original deletes leaves the deletion without +// effect, whatever it changes in the file: a port that keeps what the line +// made of it, or the file with a line added, is a pull request of its own. +func TestFoldRefusesAChangedKeptDeletion(t *testing.T) { + for name, kept := range map[string]string{ + "the line's own content": "line only\n", + "the file, one line more": "feature\nline only\n# reviewed\n", + } { + t.Run(name, func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a feature", "", map[string]string{"app/x.txt": "feature\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only setting beside the feature", map[string]string{"app/x.txt": "feature\nline only\n"}) + mc5 := f.pr(5, "fix(app): drop the feature, and five", "", map[string]string{"app/x.txt": "", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/x.txt": kept, "app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): drop the feature, and five (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/x.txt" without effect`) { + t.Errorf("fold = %d, %q, want it refused\n%s", code, out, stderr) + } + }) + } +} diff --git a/cmd/backport-batch/github.go b/cmd/backport-batch/github.go index b77ac0945..7161ef15b 100644 --- a/cmd/backport-batch/github.go +++ b/cmd/backport-batch/github.go @@ -21,12 +21,21 @@ import ( "fmt" "regexp" "strconv" + "strings" ) // openPullsCap is the page size of the open pull request listing. The REST // listing stops at it in silence, so a full page is an error, not an answer. const openPullsCap = 100 +// recentPullsPage is the page size of the listing of every pull request +// against a line, newest first, and recentPullsPages how many pages of it a +// search for the line's record reads. +const ( + recentPullsPage = 100 + recentPullsPages = 10 +) + // searchResultCap is the most results GitHub search returns for one query, // however far it is paginated. gh pr list answers a --label filter through // search, so a listing that long has to be taken as cut short. @@ -43,6 +52,7 @@ var ( batchBranchRE = regexp.MustCompile(`^backport-batch/(release-\d+\.\d+)-(\d{8})$`) loginRE = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$`) botLoginRE = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?\[bot\]$`) + pullURLRE = regexp.MustCompile(`^https://[^/\s]+/[^/\s]+/[^/\s]+/pull/([1-9][0-9]{0,8})$`) ) // batchBotEnv names the GitHub App bot the scheduled batch opens its pull @@ -66,10 +76,12 @@ func batchBotLogin(lookup func(string) (string, bool)) string { // pullRequest is the REST view of a pull request: unlike gh pr list, it // carries the author's association with the repository. type pullRequest struct { - Number int `json:"number"` - HTMLURL string `json:"html_url"` - Body string `json:"body"` - AuthorAssociation string `json:"author_association"` + Number int `json:"number"` + HTMLURL string `json:"html_url"` + Body string `json:"body"` + State string `json:"state"` + MergedAt *string `json:"merged_at"` + AuthorAssociation string `json:"author_association"` User struct { Login string `json:"login"` Type string `json:"type"` @@ -102,13 +114,22 @@ type issueComment struct { // github is the part of GitHub the tool talks to: gh outside tests. type github interface { openPulls(base string) ([]pullRequest, error) + // recentPulls is one page of every pull request against base, open, + // merged or closed, newest first. + recentPulls(base string, page int) ([]pullRequest, error) // permission is login's permission on the repository as GitHub's // legacy field gives it: admin, write, read or none. permission(login string) (string, error) // labelled lists the pull requests merged into main that carry label. labelled(label string) ([]int, error) - createPull(base, head, title, bodyFile string) error + // pull reads one pull request as it is now. + pull(number int) (pullRequest, error) + // createPull opens the draft and returns its number and URL. + createPull(base, head, title, bodyFile string) (int, string, error) editPull(number int, title, bodyFile string) error + // editBody replaces the body of pull request number and nothing else. + editBody(number int, bodyFile string) error + closePull(number int) error comments(number int) ([]issueComment, error) createComment(number int, bodyFile string) error editComment(id int64, bodyFile string) error @@ -136,6 +157,38 @@ func (ghCLI) openPulls(base string) ([]pullRequest, error) { return prs, nil } +func (ghCLI) recentPulls(base string, page int) ([]pullRequest, error) { + out, err := capture("gh", "api", "-X", "GET", "repos/{owner}/{repo}/pulls", + "-f", "state=all", "-f", "base="+base, "-f", "sort=created", "-f", "direction=desc", + "-f", "per_page="+strconv.Itoa(recentPullsPage), "-f", "page="+strconv.Itoa(page)) + if err != nil { + return nil, err + } + var prs []pullRequest + if err := json.Unmarshal([]byte(out), &prs); err != nil { + return nil, fmt.Errorf("reading the pull requests against %s: %w", base, err) + } + return prs, nil +} + +// permCache asks GitHub for a login's permission once per command: the open +// batch's author is asked when it is chosen and again when its record is. +type permCache struct { + github + perms map[string]string +} + +func (c *permCache) permission(login string) (string, error) { + if p, ok := c.perms[login]; ok { + return p, nil + } + p, err := c.github.permission(login) + if err == nil { + c.perms[login] = p + } + return p, err +} + func (ghCLI) permission(login string) (string, error) { if !loginRE.MatchString(login) { return "", fmt.Errorf("%q is not a GitHub login", login) @@ -175,9 +228,33 @@ func (ghCLI) labelled(label string) ([]int, error) { return numbers, nil } -func (ghCLI) createPull(base, head, title, bodyFile string) error { - _, err := capture("gh", "pr", "create", "--draft", "--base", base, "--head", head, "--title", title, "--body-file", bodyFile) - return err +func (ghCLI) pull(number int) (pullRequest, error) { + var pr pullRequest + out, err := capture("gh", "api", "repos/{owner}/{repo}/pulls/"+strconv.Itoa(number)) + if err != nil { + return pr, err + } + if err := json.Unmarshal([]byte(out), &pr); err != nil { + return pr, fmt.Errorf("reading pull request #%d: %w", number, err) + } + if pr.Number != number { + return pr, fmt.Errorf("asked for pull request #%d, GitHub answered with #%d", number, pr.Number) + } + return pr, nil +} + +func (ghCLI) createPull(base, head, title, bodyFile string) (int, string, error) { + out, err := capture("gh", "pr", "create", "--draft", "--base", base, "--head", head, "--title", title, "--body-file", bodyFile) + if err != nil { + return 0, "", err + } + url := strings.TrimSpace(out) + m := pullURLRE.FindStringSubmatch(url) + if m == nil { + return 0, "", fmt.Errorf("gh pr create printed %q, not the URL of a pull request", url) + } + n, err := strconv.Atoi(m[1]) + return n, url, err } func (ghCLI) editPull(number int, title, bodyFile string) error { @@ -185,6 +262,16 @@ func (ghCLI) editPull(number int, title, bodyFile string) error { return err } +func (ghCLI) editBody(number int, bodyFile string) error { + _, err := capture("gh", "pr", "edit", strconv.Itoa(number), "--body-file", bodyFile) + return err +} + +func (ghCLI) closePull(number int) error { + _, err := capture("gh", "pr", "close", strconv.Itoa(number)) + return err +} + func (ghCLI) comments(number int) ([]issueComment, error) { out, err := capture("gh", "api", "--paginate", "--slurp", "repos/{owner}/{repo}/issues/"+strconv.Itoa(number)+"/comments") if err != nil { diff --git a/cmd/backport-batch/history.go b/cmd/backport-batch/history.go index 74f4c66af..a530aa244 100644 --- a/cmd/backport-batch/history.go +++ b/cmd/backport-batch/history.go @@ -205,6 +205,9 @@ type history struct { picks []pickRef batches map[string]string backports map[int]string + // trees caches what a net merge of an original onto head writes, by its + // merge commit: "" for a conflict. + trees map[string]string } func readHistory(head, base string, textUntil time.Time) (*history, error) { @@ -225,7 +228,7 @@ func readHistory(head, base string, textUntil time.Time) (*history, error) { // readPicks reads only the messages of head's commits outside base, which is // all a batch branch is asked about; textUntil is the batch-only day. func readPicks(head, base string, textUntil time.Time) (*history, error) { - h := &history{head: head, reachable: map[string]bool{}, appended: map[string]string{}, batches: map[string]string{}, backports: map[int]string{}} + h := &history{head: head, reachable: map[string]bool{}, appended: map[string]string{}, batches: map[string]string{}, backports: map[int]string{}, trees: map[string]string{}} oids, err := revList("--end-of-options", head, "^"+base) if err != nil { return nil, err @@ -332,34 +335,54 @@ func (h *history) carries(o *original, commits []string) (string, error) { if c, ok := h.backports[o.number]; ok { return `named by "Backport of" in ` + c[:12], nil } - noop, err := noopPick(h.head, o) + tree, err := h.netTree(o) if err != nil { return "", err } - if noop { + headTree, err := resolveTree(h.head) + if err != nil { + return "", err + } + if tree == headTree { return "a pick would change nothing", nil } return "", nil } -// noopPick reports whether cherry-picking the original onto head would leave -// head's tree as it is: the content is there, whatever names it. -func noopPick(head string, o *original) (bool, error) { +// netTree is noopPick's tree for o onto the branch's head, read once. +func (h *history) netTree(o *original) (string, error) { + if tree, ok := h.trees[o.merge]; ok { + return tree, nil + } + tree, _, err := noopPick(h.head, o) + if err == nil { + h.trees[o.merge] = tree + } + return tree, err +} + +// noopPick merges the original onto head as one change and returns the tree +// that writes for a clean merge, "" for a conflict, and whether the tree is +// head's own: the content is there, whatever names it. +func noopPick(head string, o *original) (string, bool, error) { stdout, stderr, code, err := gitStatus("merge-tree", "--write-tree", "--no-messages", "--merge-base="+o.merge+"^1", "--end-of-options", head, o.merge) switch { case err != nil: - return false, err + return "", false, err case code == 1: - return false, nil + return "", false, nil case code != 0: - return false, fmt.Errorf("git merge-tree for #%d: exit %d: %s", o.number, code, strings.TrimSpace(stderr)) + return "", false, fmt.Errorf("git merge-tree for #%d: exit %d: %s", o.number, code, strings.TrimSpace(stderr)) } tree, _, _ := strings.Cut(stdout, "\n") + if err := checkSHA(tree); err != nil { + return "", false, fmt.Errorf("git merge-tree for #%d: %w", o.number, err) + } headTree, err := resolveTree(head) if err != nil { - return false, err + return "", false, err } - return tree == headTree, nil + return tree, tree == headTree, nil } func resolveTree(commit string) (string, error) { diff --git a/cmd/backport-batch/history_test.go b/cmd/backport-batch/history_test.go index f753c258c..299016c61 100644 --- a/cmd/backport-batch/history_test.go +++ b/cmd/backport-batch/history_test.go @@ -202,6 +202,9 @@ func TestCarriedByTheBatch(t *testing.T) { {name: "the batch's bot", edit: func(pr *pullRequest, gh *fakeGH) { gh.bot = "cozystack-ci[bot]" pr.User.Login, pr.User.Type, pr.AuthorAssociation = "cozystack-ci[bot]", "Bot", "CONTRIBUTOR" + // The bot writes the record into every body it opens. + l, _ := parseLine("release-1.1") + pr.Body = deferredBlock(l, nil, nil, nil) }, carried: true, notAsked: true}, {name: "the batch's bot when no bot is configured", edit: func(pr *pullRequest, _ *fakeGH) { pr.User.Login, pr.User.Type = "cozystack-ci[bot]", "Bot" diff --git a/cmd/backport-batch/lines.go b/cmd/backport-batch/lines.go index 2e3c618f2..275be00f9 100644 --- a/cmd/backport-batch/lines.go +++ b/cmd/backport-batch/lines.go @@ -207,29 +207,35 @@ func mainMerges(base, mainHead string) ([]*original, []string, error) { var merges []*original var other []string for _, m := range list { - // The subject is the first paragraph, its lines joined, as git reads it. - msg := strings.TrimRight(msgs[m.oid].message, "\n") - subject, body, _ := strings.Cut(msg, "\n\n") - subject = strings.ReplaceAll(subject, "\n", " ") o := &original{merge: m.oid, parents: m.parents, merged: msgs[m.oid].committed} - if pr := mergePRRE.FindStringSubmatch(subject); pr != nil { - o.number, _ = strconv.Atoi(pr[1]) - body = strings.TrimLeft(body, "\n") - o.title, body, _ = strings.Cut(body, "\n") - } else if pr := prNumberRE.FindStringSubmatchIndex(subject); pr != nil { - o.number, _ = strconv.Atoi(subject[pr[2]:pr[3]]) - o.title = strings.TrimSpace(subject[:pr[0]]) - } + o.number, o.title, o.body = mergedPR(msgs[m.oid].message) if o.number <= 0 || o.parents > 2 { other = append(other, m.oid) continue } - o.body = strings.TrimLeft(body, "\n") merges = append(merges, o) } return merges, other, nil } +// mergedPR reads the pull request a commit on main's first-parent line +// merged from its message, with its title and body; 0 when it names none. +func mergedPR(message string) (int, string, string) { + // The subject is the first paragraph, its lines joined, as git reads it. + subject, body, _ := strings.Cut(strings.TrimRight(message, "\n"), "\n\n") + subject = strings.ReplaceAll(subject, "\n", " ") + number, title := 0, "" + if pr := mergePRRE.FindStringSubmatch(subject); pr != nil { + number, _ = strconv.Atoi(pr[1]) + body = strings.TrimLeft(body, "\n") + title, body, _ = strings.Cut(body, "\n") + } else if pr := prNumberRE.FindStringSubmatchIndex(subject); pr != nil { + number, _ = strconv.Atoi(subject[pr[2]:pr[3]]) + title = strings.TrimSpace(subject[:pr[0]]) + } + return number, title, strings.TrimLeft(body, "\n") +} + // prCommits lists what an original brings to a line, one pick each: for a // merged pull request every commit of its own, oldest first, leaving out the // merges that brought main into its branch, whose content main already has; diff --git a/cmd/backport-batch/main.go b/cmd/backport-batch/main.go index f93282daf..23b357dff 100644 --- a/cmd/backport-batch/main.go +++ b/cmd/backport-batch/main.go @@ -34,10 +34,13 @@ Usage: backport-batch run --line release-X.Y [--onto SHA] [--date YYYY-MM-DD] [--no-workflow-files] [--push] [--policy REF] backport-batch run --line release-X.Y --continue [--no-workflow-files] [--push] [--policy REF] backport-batch run --line release-X.Y --defer N[,N] --reason TEXT [--push] [--policy REF] + backport-batch run --line release-X.Y --drop N[,N] --reason TEXT [--push] [--policy REF] + backport-batch run --line release-X.Y --undrop N[,N] [--push] [--policy REF] backport-batch check --line release-X.Y [--cut] [--policy REF] backport-batch fold --line release-X.Y --original N --commit SHA [--policy REF] backport-batch verify --line release-X.Y [--head SHA] backport-batch notify --line release-X.Y [--mention org/team|login] [--push] + backport-batch report --pr N [--id ID] [--json] backport-batch lines [--policy REF] run Builds backport-batch/- in this working tree, off @@ -53,7 +56,13 @@ run Builds backport-batch/- in this working tree, off --defer rebuilds the open batch off origin/ without those originals and records them as deferred, for --reason: the batch keeps them out, the next one tries them again. --push replaces the branch, - leased on the head the rebuild started from. + leased on the head the rebuild started from; when nothing is left it + closes the batch PR instead, after recording why in it. + --drop drops originals from the line for --reason: the line's record + keeps them out of this batch and every later one. One the open batch + carries is taken off it as --defer would; otherwise only the record + changes. --undrop withdraws a drop. Both need an open batch PR whose + record they change. check Exits 1 while a change the policy takes is neither on the line nor on the open batch PR, 0 when nothing is, 2 when it cannot tell. With --cut, the gate before an rc or a patch is cut, it also exits 1 while a change @@ -69,11 +78,18 @@ notify Prints what waits on the open batch PR for a maintainer: a batch open for 3 days or more, its deferred conflicts, and the originals that change workflows. --push keeps one comment on the PR that says what waits, mentioning --mention. +report Prints a report batch PR N keeps: the one its body carries, or the + one --id names, read from its comments or its body. --json prints the + report as it is stored. It writes nothing, so it reads merged and + closed batches too. lines Prints the release lines the batch runs for, newest first. The policy is docs/release/backports.md. run, check, fold and lines read its policy block from that page on origin/main, or on --policy REF. Every command -but notify, which reads only GitHub, fetches origin first. +but notify and report fetches origin first. notify reads only GitHub; report +reads GitHub, and for a report that keeps drops also this clone's history, +which has to hold the main and line heads the report was made from: report +never fetches on its own. BACKPORT_BATCH_BOT names the bot the scheduled batch writes as, [bot], cozystack-ci[bot] when unset: a batch pull request or a notify @@ -93,15 +109,18 @@ type app struct { now func() time.Time // policyRef is where the policy page is read from; origin/main when empty. policyRef string + // tool is the commit backport-batch was built from, for the record. + tool string } func main() { - a := &app{gh: ghCLI{bot: batchBotLogin(os.LookupEnv)}, stdout: os.Stdout, stderr: os.Stderr, now: time.Now} + a := &app{gh: ghCLI{bot: batchBotLogin(os.LookupEnv)}, stdout: os.Stdout, stderr: os.Stderr, now: time.Now, tool: toolVersion()} os.Exit(a.main(os.Args[1:])) } func (a *app) main(args []string) int { a.stdout, a.stderr = textWriter{a.stdout}, textWriter{a.stderr} + a.gh = &permCache{github: a.gh, perms: map[string]string{}} if len(args) == 0 { fmt.Fprint(a.stderr, usageText) return 2 @@ -120,6 +139,8 @@ func (a *app) main(args []string) int { return a.verify(args[1:]) case "notify": return a.notify(args[1:]) + case "report": + return a.report(args[1:]) case "lines": return a.lines(args[1:]) case "-h", "--help", "help": @@ -186,10 +207,22 @@ type verdict struct { // and by load for anything else. commits []string changes []fileChange - // onBatch is the commit of the open batch pull request that picks it. + // onBatch is the commit of the open batch pull request that picks it, + // and batched the unit of the open batch that carries it whatever the + // rules now say of it. onBatch string + batched string // label is the maintainer override the original carries, if any. label string + // partial lists what its clean net merge onto the line would leave + // without effect, and park why the batch leaves it out, if it does. + partial []string + park *parkedEntry + // deps is what it builds on that the line lacks. + deps []dependency + // diag is what could not be judged about it, shown whatever became of + // it. + diag []string } // batch is everything read about a line before anything is written. @@ -198,16 +231,50 @@ type batch struct { newer *line lineHead string mainHead string - base string - other []string - verdicts []*verdict - pr *pullRequest - prHead string + // newerHead is the newer line's head, and policyAt the commit the + // policy was read from. + newerHead string + policyAt string + base string + other []string + verdicts []*verdict + pr *pullRequest + prHead string + // picks is what the open batch pull request carries beyond the line. + picks *history + // src is the line's record and where it was found, and drops its drops + // that name an original of this batch. + src recordSource + drops map[int]droppedEntry + // lineHist and newerHist are what the line and the newer line carry, + // read once for every original judged. + lineHist, newerHist *history // start is when the line started under the policy, zero for its fork point. start time.Time // textUntil is the batch-only day: what a commit's text says counts as // evidence only on commits from before it. textUntil time.Time + // partialUnits are the originals whose unit on the open batch leaves + // what it lists without effect. + partialUnits map[*verdict][]string + // What one command read once and asks again: renames by commit pair, + // units by batch head, and how often git merge-file ran. + renameCache map[string]renameSet + unitCache map[string]map[*verdict]string + mergeFiles int + // renameLimit is the -l rename detection runs with; 0, git's widest. + renameLimit int + // brought maps main's commits since the fork point to the first-parent + // commit that brought each, and ancestors says which commits blame + // named are older than the fork point. + brought map[string]string + ancestors map[string]bool + // detecting is how long judging partial units and dependencies took. + detecting time.Duration + depCache map[string][]dependency + // lineCarries says, for a prerequisite the rules keep out, whether the + // line carries it all the same. + lineCarries map[*verdict]bool } // fetch moves to the top of the working tree first: every path the batch @@ -224,18 +291,20 @@ func fetch() error { return err } -func (a *app) policy(mainHead string) (*policy, error) { +// policy reads the policy and says from which commit. +func (a *app) policy(mainHead string) (*policy, string, error) { at := mainHead if a.policyRef != "" { if strings.HasPrefix(a.policyRef, "-") { - return nil, fmt.Errorf("--policy %q is not a ref", a.policyRef) + return nil, "", fmt.Errorf("--policy %q is not a ref", a.policyRef) } var err error if at, err = resolve(a.policyRef); err != nil { - return nil, err + return nil, "", err } } - return readPolicy(at) + pol, err := readPolicy(at) + return pol, at, err } // prepare reads the line, main and the open batch pull request, each pinned to @@ -246,7 +315,7 @@ func (a *app) prepare(l line) (*batch, error) { if b.mainHead, err = resolve("refs/remotes/" + remote + "/main"); err != nil { return nil, err } - pol, err := a.policy(b.mainHead) + pol, policyAt, err := a.policy(b.mainHead) if err != nil { return nil, err } @@ -260,7 +329,7 @@ func (a *app) prepare(l line) (*batch, error) { if b.lineHead, err = resolve("refs/remotes/" + remote + "/" + l.branch()); err != nil { return nil, err } - b.start, b.textUntil = pol.starts[l.branch()], pol.batchOnly + b.start, b.textUntil, b.policyAt = pol.starts[l.branch()], pol.batchOnly, policyAt out, err := git("merge-base", "--end-of-options", b.mainHead, b.lineHead) if err != nil { return nil, fmt.Errorf("%s shares no history with main: %w", l.branch(), err) @@ -271,17 +340,14 @@ func (a *app) prepare(l line) (*batch, error) { return nil, err } b.other = other - lineHist, err := readHistory(b.lineHead, b.mainHead, b.textUntil) - if err != nil { + if b.lineHist, err = readHistory(b.lineHead, b.mainHead, b.textUntil); err != nil { return nil, err } - var newerHist *history if b.newer != nil { - newerHead, err := resolve("refs/remotes/" + remote + "/" + b.newer.branch()) - if err != nil { + if b.newerHead, err = resolve("refs/remotes/" + remote + "/" + b.newer.branch()); err != nil { return nil, err } - if newerHist, err = readHistory(newerHead, b.mainHead, b.textUntil); err != nil { + if b.newerHist, err = readHistory(b.newerHead, b.mainHead, b.textUntil); err != nil { return nil, err } } @@ -300,29 +366,51 @@ func (a *app) prepare(l line) (*batch, error) { if err != nil { return nil, err } - var prPicks *history if pr != nil { b.pr = pr if b.prHead, err = batchHead(pr); err != nil { return nil, err } - if prPicks, err = readPicks(b.prHead, b.lineHead, b.textUntil); err != nil { + if b.picks, err = readPicks(b.prHead, b.lineHead, b.textUntil); err != nil { return nil, err } } + if b.src, err = findLineRecord(a.gh, l, pr); err != nil { + return nil, err + } + if b.drops, err = b.bindDrops(merges); err != nil { + return nil, err + } + if err := b.waitOrder(merges); err != nil { + return nil, err + } for _, o := range merges { - v, err := judge(o, labels[o.number], b.start, lineHist, b.newer, newerHist) - if err != nil { + v := &verdict{o: o, label: labels[o.number]} + if err := b.judge(v, b.drops); err != nil { return nil, err } - if v.rule == 0 && prPicks != nil { - v.onBatch, _ = prPicks.found(v.o, v.commits) - } b.verdicts = append(b.verdicts, v) } + if err := b.partialStates(); err != nil { + return nil, err + } return b, nil } +// judge judges v again under drops, and finds it on the open batch when the +// policy takes it. +func (b *batch) judge(v *verdict, drops map[int]droppedEntry) error { + j, err := judge(v.o, v.label, b.start, b.lineHist, b.newer, b.newerHist, drops) + if err != nil { + return err + } + *v = *j + if v.rule == 0 && b.picks != nil { + v.onBatch, _ = b.picks.found(v.o, v.commits) + } + return nil +} + // overrides maps each pull request merged into main that carries a backport/ // label to the one that decides; backport/skip wins over backport/include. func overrides(gh github) (map[int]string, error) { @@ -344,8 +432,9 @@ func overrides(gh github) (map[int]string, error) { // started under the policy after it branched takes nothing merged before that // start. backport/skip keeps an original out whatever the rules say, and // backport/include lifts the start and rules 2 to 4, but not what the line or -// the newer line carries. -func judge(o *original, label string, start time.Time, lineHist *history, newer *line, newerHist *history) (*verdict, error) { +// the newer line carries. A maintainer's drop keeps out what the line does +// not carry, include or not. +func judge(o *original, label string, start time.Time, lineHist *history, newer *line, newerHist *history, drops map[int]droppedEntry) (*verdict, error) { v := &verdict{o: o, label: label} if label == labelSkip { v.rule, v.reason = ruleSkipLabel, labelSkip @@ -388,6 +477,10 @@ func judge(o *original, label string, start time.Time, lineHist *history, newer v.rule, v.reason = ruleOnLine, how return v, nil } + if d, ok := drops[o.number]; ok { + v.rule, v.reason = ruleDropped, fmt.Sprintf("dropped on #%d: %s", d.On, d.Reason) + return v, nil + } if newer != nil { how, err := newerHist.carries(o, v.commits) if err != nil { diff --git a/cmd/backport-batch/notes_test.go b/cmd/backport-batch/notes_test.go new file mode 100644 index 000000000..35f6ccbb4 --- /dev/null +++ b/cmd/backport-batch/notes_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os/exec" + "regexp" + "slices" + "strings" + "testing" +) + +// releaseNoteBlock is the release-note block of a body. +func releaseNoteBlock(t *testing.T, body string) string { + t.Helper() + m := regexp.MustCompile("(?s)```release-note\n(.*?)\n```").FindStringSubmatch(body) + if m == nil { + t.Fatalf("no release-note block in:\n%s", body) + } + return m[1] +} + +// A pick that differs from its original keeps its release note out of the +// release-note block, under Check these notes with what to compare: one git +// had to fit to a line that already had part of the change, and one whose +// first commit came out empty because the line had resized the value itself. +func TestChangedOriginalNoteIsSetApart(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): two, on the line", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "fix(chart): two replicas, on the line", map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}) + f.pr(5, "fix(app): two and four", "```release-note\nfix(app): two and four are loud.\n```\n", map[string]string{"app/a.txt": strings.NewReplacer("two", "TWO", "four", "FOUR").Replace(appFile)}) + f.prCommits(6, "fix(chart): two replicas and six", "```release-note\nfix(chart): two replicas by default.\n```\n", + change{files: map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}}, + change{files: map[string]string{"app/six.txt": "6\n"}}) + f.pr(7, "fix(app): seven", "```release-note\nfix(app): seven is calm.\n```\n", map[string]string{"app/seven.txt": "7\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): two and four (#5)", "fix(chart): two replicas and six (#6)", "fix(app): seven (#7)"}) { + t.Fatalf("picked %q, want #5, #6 and #7", got) + } + notes := releaseNoteBlock(t, stdout) + if !strings.Contains(notes, "seven is calm") || strings.Contains(notes, "two and four are loud") || strings.Contains(notes, "two replicas by default") { + t.Errorf("the release-note block keeps a changed pick's note, or loses #7's:\n%s", notes) + } + _, check, ok := strings.Cut(stdout, "### Check these notes") + check, _, _ = strings.Cut(check, "\n#") + if !ok || !strings.Contains(check, "two and four are loud") || !strings.Contains(check, "two replicas by default") || strings.Contains(check, "seven is calm") { + t.Errorf("Check these notes does not list #5 and #6 alone:\n%s", stdout) + } + if !strings.Contains(stdout, `"changed":[5,6]`) { + t.Errorf("the record or the report does not mark #5 and #6 changed:\n%s", stdout) + } +} + +// The command printed for a changed pick pairs the pick with the commit it +// came from, however much the pick lost: the default creation factor leaves +// a pick that carries only #5088's test unpaired. +func TestRangeDiffCommandPairsThePick(t *testing.T) { + f, mc5 := policyWorld(t) + gh := &fakeGH{} + head := handBatch(f, gh, mc5) + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue") + if code != 0 { + t.Fatalf("run --continue = %d\n%s", code, stderr) + } + m := regexp.MustCompile("`git range-diff --creation-factor=999 ([0-9a-f]{12})\\^! ([0-9a-f]{12})\\^!`").FindStringSubmatch(stdout) + if m == nil { + t.Fatalf("the body prints no range-diff for #5's pick:\n%s", stdout) + } + src, pick := f.git("rev-parse", mc5+"^2"), f.git("rev-parse", head+"^2") + if !strings.HasPrefix(src, m[1]) || !strings.HasPrefix(pick, m[2]) { + t.Errorf("the range-diff names %s and %s, want #5's commit %.12s and its pick %.12s", m[1], m[2], src, pick) + } + out, err := exec.Command("git", "range-diff", "--no-color", "--creation-factor=999", m[1]+"^!", m[2]+"^!").Output() + if want := "1: " + f.git("rev-parse", "--short", src) + " ! 1: " + f.git("rev-parse", "--short", pick); err != nil || !strings.HasPrefix(string(out), want) { + t.Errorf("the printed range-diff gives %q, %v; want it to start %q", out, err, want) + } + if out := f.git("range-diff", "--no-color", mc5+"^1.."+mc5+"^2", head+"^1.."+head+"^2"); !strings.Contains(out, " < ") { + t.Errorf("the default creation factor pairs #5's pick after all, so the test shows nothing:\n%s", out) + } +} diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go index 87b3a123f..fe301a2ff 100644 --- a/cmd/backport-batch/notify.go +++ b/cmd/backport-batch/notify.go @@ -70,6 +70,15 @@ func (a *app) notify(args []string) int { if err != nil { return a.fail(err) } + // With no batch open there is nothing to report on and nowhere to write, + // but a record nobody can read is said all the same. + src, err := findLineRecord(a.gh, l, pr) + if err != nil { + return a.fail(err) + } + if c := src.closingOpen(); c != nil { + return a.fail(fmt.Errorf("batch pull request #%d is being closed; backport-batch run --line %s --push finishes it", c.PR, l.branch())) + } if pr == nil { return 0 } @@ -77,21 +86,49 @@ func (a *app) notify(args []string) int { if err != nil { return a.fail(err) } - deferred, err := parseDeferred(pr.Body, l) - if err != nil { - return a.fail(fmt.Errorf("batch pull request #%d: %w", pr.Number, err)) + deferred := map[int]deferredEntry{} + var partial []int + var parks []parkedEntry + if src.open { + deferred, partial = src.rec.deferred(), src.rec.Partial + for _, p := range src.rec.Parked { + if !p.Announced { + parks = append(parks, p) + } + } } + // An original that waits on others is listed once, after what decides + // its wait, where that one is listed. var conflicts, waiting []int + after := map[int][]int{} for n, e := range deferred { - switch { - case e.Workflows: + switch place := placement(deferred, n, map[int]bool{}); { + case place == "workflow": waiting = append(waiting, n) - case e.Reason == "": + case place == "conflict": conflicts = append(conflicts, n) } + if len(e.Needs) > 0 { + after[n] = e.Needs + } } slices.Sort(conflicts) slices.Sort(waiting) + item := func(n int) string { + if needs := after[n]; len(needs) > 0 { + return fmt.Sprintf("\n- %s, after %s", prLink(n), prLinks(needs)) + } + return "\n- " + prLink(n) + } + count := func(ns []int) int { + c := 0 + for _, n := range ns { + if len(after[n]) == 0 { + c++ + } + } + return c + } today := a.now().UTC().Truncate(24 * time.Hour) days := int(today.Sub(opened).Hours() / 24) @@ -101,28 +138,135 @@ func (a *app) notify(args []string) int { fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days)) } - if len(conflicts) > 0 { - fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), len(conflicts), pr.Number) + if len(partial) > 0 { + fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number) var b strings.Builder - fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch:\n", tasksURL) + b.WriteString("These are on the batch but carry only part of their originals on this line; take each out with `backport-batch run --defer` before the cut, and port what applies as a pull request of its own:\n") + for _, n := range partial { + b.WriteString(item(n)) + } + text = append(text, b.String()) + } + if len(conflicts) > 0 { + fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number) + var b strings.Builder + fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch, and so does what waits on one:\n", tasksURL) for _, n := range conflicts { - fmt.Fprintf(&b, "\n- %s", prLink(n)) + b.WriteString(item(n)) } text = append(text, b.String()) } if len(waiting) > 0 { - fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + if count(waiting) == len(waiting) { + fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + } else { + fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir) + } var b strings.Builder - fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes:\n", workflowsDir, policyURL) + fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes, or wait on one that does:\n", workflowsDir, policyURL) for _, n := range waiting { - fmt.Fprintf(&b, "\n- %s", prLink(n)) + b.WriteString(item(n)) + } + text = append(text, b.String()) + } + if len(parks) > 0 { + fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number) + var b strings.Builder + fmt.Fprintf(&b, "These cannot land on this line whole, so the batch parked them; each is named here once. A maintainer ports what applies as a pull request of its own, as [fixes that exist only on a release line](%s#fixes-that-exist-only-on-a-release-line) describes, or drops the original with `backport-batch run --drop`:\n", policyURL) + for _, p := range parks { + fmt.Fprintf(&b, "\n- %s: %s", prLink(p.Original), parkReason(p)) } text = append(text, b.String()) } if !*push { return 0 } - return a.comment(pr, *mention, today, text) + if code := a.comment(pr, *mention, today, text); code != 0 || len(parks) == 0 { + return code + } + if err := a.announced(pr, src.rec, parks); err != nil { + return a.fail(fmt.Errorf("%w; the next notify names them again", err)) + } + return 0 +} + +// announced marks parks as named in the record of the open batch pull +// request, which the comment just named: only while it is open and its body +// still carries the record notify read, so a run that rewrote the body +// meanwhile loses nothing; the next notify names those parks again. +func (a *app) announced(pr *pullRequest, rec *lineRecord, parks []parkedEntry) error { + marked := *rec + marked.Parked = slices.Clone(rec.Parked) + for i, p := range marked.Parked { + if slices.ContainsFunc(parks, func(q parkedEntry) bool { return q.Original == p.Original && parkEvent(q) == parkEvent(p) }) { + marked.Parked[i].Announced = true + } + } + now, err := a.gh.pull(pr.Number) + if err != nil { + return err + } + if st := prState(&now); st != "open" { + return fmt.Errorf("batch pull request #%d is %s now; nothing is written to it", pr.Number, st) + } + lines, err := fencedLines(now.Body, deferredFence) + if err != nil || len(lines) != 1 { + return fmt.Errorf("batch pull request #%d no longer carries the record notify read", pr.Number) + } + read, err := parseRecord(now.Body, notifyLineOf(rec)) + if err != nil || recordBlock(*read) != recordBlock(*rec) { + return fmt.Errorf("batch pull request #%d no longer carries the record notify read", pr.Number) + } + // GitHub keeps the CRLF line ends a body edited on the web has; the + // record is found, and the body written back, with LF ones. + body := strings.ReplaceAll(now.Body, "\r\n", "\n") + block := deferredFence + "\n" + lines[0] + "\n```" + if strings.Count(body, block) != 1 { + return fmt.Errorf("batch pull request #%d carries the record notify read, but not as one block it can mark", pr.Number) + } + body = strings.Replace(body, block, recordBlock(marked), 1) + name, err := writeTemp("backport-batch-body-*.md", body) + if err != nil { + return err + } + defer os.Remove(name) + return a.gh.editBody(pr.Number, name) +} + +// notifyLineOf is the line a record is for, which it was read against. +func notifyLineOf(rec *lineRecord) line { + l, _ := parseLine(rec.Line) + return l +} + +// placement says under which list notify names original n of the record: +// workflow for a wait for a maintainer, conflict for a deferred conflict, "" +// for neither, a deferral for a reason among them. One that waits on others +// goes where what decides its wait goes, followed down the record; one that +// is not in the record is held by rule 6 or missing, and waits on nobody. +func placement(deferred map[int]deferredEntry, n int, seen map[int]bool) string { + e, ok := deferred[n] + switch { + case !ok || seen[n]: + return "" + case e.Workflows: + return "workflow" + case e.Reason != "": + return "" + case len(e.Needs) == 0: + return "conflict" + } + seen[n] = true + place := "" + for _, f := range e.Needs { + switch p := placement(deferred, f, seen); { + case p == "workflow": + return p + case p == "conflict": + place = p + } + } + return place } // comment writes what waits into the one notify comment on the batch pull @@ -173,6 +317,9 @@ func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []s if err := f.Close(); err != nil { return a.fail(err) } + if err := a.stillOpen(pr.Number); err != nil { + return a.fail(err) + } if ours != nil { err = a.gh.editComment(ours.ID, f.Name()) } else { @@ -198,7 +345,7 @@ func (a *app) lines(args []string) int { if err != nil { return a.fail(err) } - pol, err := a.policy(mainHead) + pol, _, err := a.policy(mainHead) if err != nil { return a.fail(err) } diff --git a/cmd/backport-batch/notify_test.go b/cmd/backport-batch/notify_test.go index 606953be5..7f870baee 100644 --- a/cmd/backport-batch/notify_test.go +++ b/cmd/backport-batch/notify_test.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "errors" "slices" "strings" "testing" @@ -148,7 +149,8 @@ func TestNotifyFindsTheBotsComment(t *testing.T) { ours.User.Login, ours.User.Type = bot, "Bot" other := issueComment{ID: 8, Body: notifyMarker + "\nplanted"} other.User.Login, other.User.Type = "renovate[bot]", "Bot" - pr := notifyPR("20261019", "") + // The bot writes the record into every body it opens. + pr := notifyPR("20261019", deferredBlock(notifyLine, nil, nil, nil)) pr.User.Login, pr.User.Type = bot, "Bot" gh := &fakeGH{prs: []pullRequest{pr}, issueComments: []issueComment{ours, other}, bot: bot} if code, stdout, stderr := notifyRun(gh, "--push"); code != 0 || !strings.Contains(stdout, "has been open since") { @@ -172,7 +174,7 @@ func TestNotifyFailsOnAnUnreadablePermission(t *testing.T) { } const bot = "cozystack-ci[bot]" - pr := notifyPR("20261019", "") + pr := notifyPR("20261019", deferredBlock(notifyLine, nil, nil, nil)) pr.User.Login, pr.User.Type = bot, "Bot" ours := issueComment{ID: 7, Body: notifyMarker + "\nearlier"} ours.User.Login, ours.User.Type = "maintainer", "User" @@ -203,6 +205,32 @@ func TestNotifyRefuses(t *testing.T) { } } +// notify reads the line's record the way the other commands do, and stays +// silent with no batch open, but a record it cannot read, or one that says +// its open batch is being closed, stops it. +func TestNotifyValidatesTheRecord(t *testing.T) { + sha := strings.Repeat("a", 40) + closingRecord := strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":91,"branch":"backport-batch/release-1.1-20261002","head":"`+sha+`"}`, 1) + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + for _, tc := range []struct { + name string + pr pullRequest + code int + quiet bool + }{ + {"a clean closed record", recordPR(91, "closed", "maintainer", block(v2(""))), 0, true}, + {"a malformed record", recordPR(91, "closed", "maintainer", block(`{"line":`)), 2, false}, + {"a closed record that was closing", recordPR(91, "closed", "maintainer", block(closingRecord)), 0, true}, + {"an open record that is closing", recordPR(91, "open", "maintainer", block(closingRecord)), 2, false}, + } { + gh := &fakeGH{prs: []pullRequest{tc.pr}} + code, stdout, stderr := notifyRun(gh, "--push") + if code != tc.code || tc.quiet && (stdout != "" || stderr != "") || len(gh.issueComments) != 0 { + t.Errorf("%s: notify = %d, %q, %q, %d comments; want %d and nothing written", tc.name, code, stdout, stderr, len(gh.issueComments), tc.code) + } + } +} + // The maintainer's example: with 1.6 and 1.7 released, release-1.8 joins them // when it branches, and 1.8.0 drops 1.6. func TestSupportedLines(t *testing.T) { @@ -286,3 +314,183 @@ func TestRunWithNothingToPick(t *testing.T) { t.Errorf("pushed %q and opened %q with nothing picked", f.remoteHead(batchBranch1), gh.created) } } + +// notify writes its comment only after GitHub says the batch is still open, +// so a batch closed while notify reads it is not written to, whether the +// comment would be new or an update of its own. +func TestNotifyWritesNothingOnABatchClosedMeanwhile(t *testing.T) { + for _, update := range []bool{false, true} { + t.Run(map[bool]string{false: "a new comment", true: "an update"}[update], func(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261023", recordBody(5))}} + if update { + ours := issueComment{ID: 7, Body: notifyMarker + "\nearlier"} + ours.User.Login, ours.User.Type = "maintainer", "User" + gh.issueComments = []issueComment{ours} + } + gh.after = map[string]func(){"comments": func() { gh.find(90).State = "closed" }} + code, _, stderr := notifyRun(gh, "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || len(gh.closedWrites) != 0 || update && gh.issueComments[0].Body != notifyMarker+"\nearlier" { + t.Errorf("notify = %d, closed writes %q, comments %+v; want 2 and nothing written:\n%s", code, gh.closedWrites, gh.issueComments, stderr) + } + }) + } +} + +// parkedBody is an open batch body whose version 3 record parks p. +func parkedBody(p ...parkedEntry) string { + sha := strings.Repeat("a", 40) + rec := lineRecord{Version: recordVersion, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeContinue}}, + Deferred: []deferredEntry{}, Dropped: []droppedEntry{}, Stale: []int{}, Parked: p} + return "batch\n\n" + recordBlock(rec) + "\n" +} + +// notifyAt runs notify at a time of day, with gh faked. +func notifyAt(gh *fakeGH, at time.Time, args ...string) (int, string, string) { + var stdout, stderr bytes.Buffer + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return at }} + code := a.main(append([]string{"notify", "--line", "release-1.1"}, args...)) + return code, stdout.String(), stderr.String() +} + +// A park is announced once on the open batch: notify names it in its +// comment and marks it announced in the record; a later notify is silent +// about it. A mark that fails to land repeats the announcement once, the +// next day. With no batch open nothing is announced, and the first notify of +// the next batch does it. +func TestNotifyAnnouncesAParkOnce(t *testing.T) { + park := parkedEntry{Original: 5, Merge: strings.Repeat("c", 40), Kind: parkDependency, Needs: []int{3}} + parked := func(gh *fakeGH) []parkedEntry { + t.Helper() + rec, err := parseRecord(gh.prs[0].Body, notifyLine) + if err != nil { + t.Fatal(err) + } + return rec.Parked + } + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + code, stdout, stderr := notifyRun(gh, "--push") + if code != 0 || !strings.Contains(stdout, "release-1.1: 1 parked originals on #90, named once") || len(gh.issueComments) != 1 || !strings.Contains(gh.issueComments[0].Body, "- [#5](https://github.com/cozystack/cozystack/pull/5): builds on [#3](") { + t.Fatalf("notify = %d, want #5 announced:\n%s\n%s\n%v", code, stdout, stderr, gh.issueComments) + } + if p := parked(gh); len(p) != 1 || !p[0].Announced { + t.Errorf("the record parks %+v, want #5 marked announced", p) + } + if code, stdout, _ = notifyRun(gh, "--push"); code != 0 || strings.Contains(stdout, "parked") || strings.Contains(gh.issueComments[0].Body, "[#5]") { + t.Errorf("notify again = %d, want nothing about #5:\n%s\n%s", code, stdout, gh.issueComments[0].Body) + } + + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}, fail: map[string]error{"editBody": errors.New("HTTP 502")}} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || len(gh.issueComments) != 1 || parked(gh)[0].Announced { + t.Fatalf("notify with a failing mark = %d, want 2 after the comment, the record unmarked", code) + } + code, _, stderr = notifyAt(gh, time.Date(2026, 10, 27, 0, 5, 0, 0, time.UTC), "--push") + if code != 0 || !strings.Contains(gh.issueComments[0].Body, "[#5](") || !parked(gh)[0].Announced { + t.Errorf("notify the next day = %d, want #5 named once more and marked:\n%s\n%s", code, stderr, gh.issueComments[0].Body) + } + + // Closed, or rewritten by a run, between the comment and the mark: the + // mark is not written, and the next notify names #5 again. + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + gh.after = map[string]func(){"createComment": func() { gh.prs[0].State = "closed" }} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || len(gh.closedWrites) != 0 || parked(gh)[0].Announced { + t.Errorf("notify on a batch closed meanwhile = %d, writes %q; want 2 and no mark", code, gh.closedWrites) + } + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + rewritten := parkedBody(park, parkedEntry{Original: 6, Merge: strings.Repeat("d", 40), Kind: parkPartial, Files: []string{"x"}}) + gh.after = map[string]func(){"createComment": func() { gh.prs[0].Body = rewritten }} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || gh.prs[0].Body != rewritten || len(gh.bodies) != 0 { + t.Errorf("notify on a body rewritten meanwhile = %d; want 2 and the new body left as it is", code) + } + + // A body someone edited on the web comes back with CRLF line ends; the + // mark lands all the same. + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", strings.ReplaceAll(parkedBody(park), "\n", "\r\n"))}} + if code, _, stderr := notifyRun(gh, "--push"); code != 0 || !parked(gh)[0].Announced { + t.Errorf("notify on a CRLF body = %d, want #5 marked:\n%s", code, stderr) + } + + closed := notifyPR("20261026", parkedBody(park)) + at := "2026-10-26T12:00:00Z" + closed.State, closed.MergedAt = "closed", &at + gh = &fakeGH{prs: []pullRequest{closed}} + if code, stdout, _ := notifyRun(gh, "--push"); code != 0 || stdout != "" || len(gh.issueComments) != 0 || len(gh.bodies) != 0 { + t.Errorf("notify with no batch open = %d, %q, %d comments, %d edits; want nothing", code, stdout, len(gh.issueComments), len(gh.bodies)) + } + next := notifyPR("20261027", parkedBody(park)) + next.Number = 91 + gh.prs = append(gh.prs, next) + if code, _, _ := notifyRun(gh, "--push"); code != 0 || len(gh.issueComments) != 1 || !strings.Contains(gh.issueComments[0].Body, "[#5](") { + t.Errorf("the next batch's notify = %d, want #5 announced on it: %v", code, gh.issueComments) + } +} + +// A park carries its announcement into the next record only while it is +// the same park: the same merge, kind, prerequisites, commits and files. +func TestParkEventIdentity(t *testing.T) { + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + was := parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}, Announced: true} + carrier := &lineRecord{Parked: []parkedEntry{was}} + for name, tc := range map[string]struct { + now parkedEntry + announced bool + }{ + "the same park": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}}, true}, + "another unnumbered commit": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{b}}, false}, + "another kind": {parkedEntry{Original: 5, Merge: a, Kind: parkPartial, Files: []string{"x"}}, false}, + "another prerequisite": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}, Needs: []int{3}}, false}, + "another merge": {parkedEntry{Original: 5, Merge: b, Kind: parkDependency, Commits: []string{a}}, false}, + "another original": {parkedEntry{Original: 6, Merge: a, Kind: parkDependency, Commits: []string{a}}, false}, + } { + got := carryAnnounced([]parkedEntry{tc.now}, carrier) + if got[0].Announced != tc.announced { + t.Errorf("%s: announced = %v, want %v", name, got[0].Announced, tc.announced) + } + } + if got := carryAnnounced([]parkedEntry{was}, nil); got[0].Announced { + t.Error("a park with no carrier came out announced") + } + unannounced := was + unannounced.Announced = false + if got := carryAnnounced([]parkedEntry{was}, &lineRecord{Parked: []parkedEntry{unannounced}}); got[0].Announced { + t.Error("a park the carrier holds unannounced came out announced") + } +} + +// A park keeps its announcement through a continuation and into the next +// batch, so notify does not name it again. +func TestRolloverKeepsTheAnnouncement(t *testing.T) { + f := featWorld(t) + gh := &fakeGH{} + openBatch(t, f, gh) + if code, _, stderr := f.exec(gh, "notify", "--line", "release-1.1", "--push"); code != 0 || !strings.Contains(notifyComment(gh), "[#5](") { + t.Fatalf("notify = %d, want #5 announced:\n%s", code, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 || !strings.Contains(stdout, `"needs":[3],"announced":true`) { + t.Fatalf("run --continue = %d, want #5's park still announced:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + mergeBatch(f, gh) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.nextPR = 91 + f.now = f.now.Add(24 * time.Hour) + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 || !strings.Contains(stdout, `"needs":[3],"announced":true`) { + t.Fatalf("the next batch's run = %d, want #5's park announced from the merged batch:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + gh.issueComments = nil + if code, stdout, _ := f.exec(gh, "notify", "--line", "release-1.1", "--push"); code != 0 || strings.Contains(stdout, "parked") || strings.Contains(notifyComment(gh), "[#5](") { + t.Errorf("notify on the next batch = %d, want nothing about #5:\n%s\n%s", code, stdout, notifyComment(gh)) + } +} + +// notifyComment is the notify comment on the fake, or "". +func notifyComment(gh *fakeGH) string { + for _, c := range slices.Backward(gh.issueComments) { + if strings.HasPrefix(c.Body, notifyMarker) { + return c.Body + } + } + return "" +} diff --git a/cmd/backport-batch/overrides_test.go b/cmd/backport-batch/overrides_test.go index f1b1a5bcb..0bcdc426e 100644 --- a/cmd/backport-batch/overrides_test.go +++ b/cmd/backport-batch/overrides_test.go @@ -196,6 +196,13 @@ func TestGHCLI(t *testing.T) { if want := "api -X GET repos/{owner}/{repo}/pulls -f state=open -f base=release-1.6 -f per_page=100"; called() != want { t.Errorf("gh %q, want gh %q", called(), want) } + write("[]") + if _, err := (ghCLI{}).recentPulls("release-1.6", 2); err != nil { + t.Fatal(err) + } + if want := "api -X GET repos/{owner}/{repo}/pulls -f state=all -f base=release-1.6 -f sort=created -f direction=desc -f per_page=100 -f page=2"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } var page []string for i := range openPullsCap { page = append(page, fmt.Sprintf(`{"number":%d}`, i+1)) @@ -230,13 +237,34 @@ func TestGHCLI(t *testing.T) { // A batch pull request carries no label: the release workflows act on a // pull request cozystack-ci[bot] opened only when it has the release label. - write("") - if err := (ghCLI{}).createPull("release-1.7", batchBranch1, "fix(backport): batch", "/tmp/body.md"); err != nil { - t.Fatal(err) + write("https://github.com/cozystack/cozystack/pull/5300\n") + if n, url, err := (ghCLI{}).createPull("release-1.7", batchBranch1, "fix(backport): batch", "/tmp/body.md"); err != nil || n != 5300 || url != "https://github.com/cozystack/cozystack/pull/5300" { + t.Errorf("createPull = %d, %q, %v; want the pull request gh printed", n, url, err) } if want := "pr create --draft --base release-1.7 --head " + batchBranch1 + " --title fix(backport): batch --body-file /tmp/body.md"; called() != want { t.Errorf("gh %q, want gh %q", called(), want) } + write("Warning: 1 uncommitted change\n") + if _, _, err := (ghCLI{}).createPull("release-1.7", batchBranch1, "fix(backport): batch", "/tmp/body.md"); err == nil { + t.Error("createPull took output that is no pull request URL") + } + write(`{"number":90,"state":"closed","merged_at":"2026-10-26T12:00:00Z","head":{"ref":"x","sha":"y"}}`) + if pr, err := (ghCLI{}).pull(90); err != nil || pr.State != "closed" || pr.MergedAt == nil { + t.Errorf("pull = %+v, %v; want #90 merged", pr, err) + } + if want := "api repos/{owner}/{repo}/pulls/90"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + if _, err := (ghCLI{}).pull(91); err == nil { + t.Error("pull took GitHub's answer about another pull request") + } + write("") + if err := (ghCLI{}).closePull(90); err != nil { + t.Fatal(err) + } + if want := "pr close 90"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } write("") if err := (ghCLI{}).editPull(90, "fix(backport): batch", "/tmp/body.md"); err != nil { t.Fatal(err) diff --git a/cmd/backport-batch/park_test.go b/cmd/backport-batch/park_test.go new file mode 100644 index 000000000..94a06ada5 --- /dev/null +++ b/cmd/backport-batch/park_test.go @@ -0,0 +1,796 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +var line11, _ = parseLine("release-1.1") + +// handBatch pushes a batch carrying merges in the batch's shape, as a run +// made one before the partial guard, and opens its pull request with a +// record that defers nothing. +func handBatch(f *fixture, gh *fakeGH, merges ...string) string { + f.t.Helper() + f.git("fetch", "-q", "origin") + f.git("switch", "-q", "--detach", "origin/release-1.1") + for _, mc := range merges { + if parents := strings.Fields(f.git("rev-list", "--parents", "-n", "1", mc)); len(parents) == 2 { + f.git("cherry-pick", "-x", mc) + continue + } + base := f.git("rev-parse", "HEAD") + for _, c := range strings.Fields(f.git("rev-list", "--reverse", "--no-merges", mc+"^1.."+mc+"^2")) { + f.git("cherry-pick", "-x", c) + } + merge := f.joined(base, "HEAD", f.git("log", "-1", "--format=%B", mc)+"\n\n"+batchTrailer+": "+mc) + f.git("update-ref", "HEAD", merge) + } + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+deferredBlock(line11, nil, nil, nil)+"\n")} + return f.remoteHead(batchBranch1) +} + +// policyWorld is #5088's shape: #3, a feat the line never gets, adds a +// policy and its test; #5 deletes both again and adds a test of its own. +func policyWorld(t *testing.T) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(3, "feat(chart): a network policy", "", map[string]string{"chart/templates/policy.yaml": "kind: NetworkPolicy\n", "chart/tests/policy_test.yaml": "suite: policy\n"}) + mc5 := f.pr(5, "fix(chart): remove the policy a rebase brought back", "```release-note\nfix(chart): the stray network policy is gone.\n```\n", + map[string]string{"chart/templates/policy.yaml": "", "chart/tests/policy_test.yaml": "", "chart/tests/values_test.yaml": "suite: values\n"}) + return f, mc5 +} + +// A clean pick that would delete files the line never had, and so carry only +// #5's test, is withheld: #5 is parked as partial, with the paths it would +// leave without effect, and its release note stays out of the body. +func TestPartialPickIsWithheld(t *testing.T) { + f, mc5 := policyWorld(t) + f.pr(6, "fix(app): six", "```release-note\nfix(app): six works.\n```\n", map[string]string{"app/six.txt": "6\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q, want only #6", got) + } + want := `"parked":[{"original":5,"merge":"` + mc5 + `","kind":"partial","files":["chart/templates/policy.yaml","chart/tests/policy_test.yaml"]}]` + if !strings.Contains(stdout, want) { + t.Errorf("the record does not park #5 as partial (%s):\n%s", want, stdout) + } + if !strings.Contains(stdout, "### Parked") || !strings.Contains(stdout, "`chart/templates/policy.yaml`") { + t.Errorf("the body does not say #5 is parked and why:\n%s", stdout) + } + if strings.Contains(stdout, "stray network policy") || !strings.Contains(stdout, "six works") { + t.Errorf("the release notes are not #6's alone:\n%s", stdout) + } +} + +// A file the line removed itself, which no original added after the line +// branched, still leaves #5's deletion without effect: check finds it with no +// record and parks it, and run picks nothing. +func TestPartialWithoutAnAdder(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: add a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(5, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); len(got) != 0 { + t.Errorf("picked %q, want nothing", got) + } + if !strings.Contains(stdout, "### Parked") { + t.Errorf("the body does not park #5:\n%s", stdout) + } + f.git("switch", "-q", "main") + code, stdout, stderr = f.exec(nil, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "note: #5 is parked: only part of it applies to release-1.1") { + t.Errorf("check = %d, want 0 with a parked-partial note:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } +} + +// The picker judges what its picks did, not what the original's merge did: +// #5's first commit deletes a file the line no longer has and comes out +// empty, its second adds a test, and the unit left would be the test alone. +func TestPartialByAnEmptyOwnCommit(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: add a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.prCommits(5, "fix(app): drop the policy and test it", "```release-note\nfix(app): no more policy.\n```\n", + change{files: map[string]string{"app/policy.yaml": ""}}, + change{files: map[string]string{"app/policy_test.txt": "gone\n"}}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); len(got) != 0 { + t.Errorf("picked %q, want #5 withheld", got) + } + if strings.Contains(stdout, "no more policy") || !strings.Contains(stdout, "### Parked") { + t.Errorf("#5's note is in the body, or #5 is not parked:\n%s", stdout) + } +} + +// A batch built before the guard can already carry a partial unit. It fails +// the cut and verify whatever its label says, and --defer takes it off, +// closing a batch it was the last unit of. +func TestExistingPartialUnitFailsTheCut(t *testing.T) { + f, mc5 := policyWorld(t) + gh := &fakeGH{} + head := handBatch(f, gh, mc5) + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is on batch #90, but only part of it applies to release-1.1") { + t.Errorf("check --cut = %d, want 1 naming the partial unit:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, stdout, stderr = f.exec(gh, "verify", "--line", "release-1.1", "--head", head) + if code != 1 || !strings.Contains(stdout, `#5 leaves "chart/templates/policy.yaml", "chart/tests/policy_test.yaml" without effect on release-1.1`) { + t.Errorf("verify = %d, want 1 naming what #5 leaves out:\n%s\n%s", code, stdout, stderr) + } + gh.labels = map[string][]int{labelSkip: {5}} + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is on batch #90, but only part of it applies to release-1.1") { + t.Errorf("check --cut with backport/skip = %d, want the partial unit to still fail it:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, _, stderr = f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "only part of it applies", "--push") + if code != 0 || prState(gh.find(90)) != "closed" || f.remoteHead(batchBranch1) != "" { + t.Errorf("run --defer 5 --push = %d, #90 %s, branch at %q; want the batch closed and its branch gone:\n%s", code, prState(gh.find(90)), f.remoteHead(batchBranch1), stderr) + } +} + +// An original whose first pick came out empty, because the line already had +// that change, is on the batch all the same: check does not call it missing, +// a continuation keeps it picked, and so does a rebuild without another +// original. +func TestOnBatchAfterAnEmptyFirstPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old\n", "app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): a, on the line first", map[string]string{"app/a.txt": "fixed\n"}) + f.prCommits(5, "fix(app): a and b", "", + change{files: map[string]string{"app/a.txt": "fixed\n"}}, + change{files: map[string]string{"app/b.txt": "fixed\n"}}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a and b (#5)", "fix(app): six (#6)"}) { + t.Fatalf("the batch carries %q, want #5 and #6", got) + } + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 0 { + t.Errorf("check = %d, want 0 with #5 on the batch:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, stdout, _ = f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is only on open batch PR #90") { + t.Errorf("check --cut = %d, want #5 only on the batch:\n%s", code, stdout) + } + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--continue") + if code != 0 || !strings.Contains(stdout, `"picked":[5,6]`) { + t.Errorf("run --continue = %d, want #5 and #6 picked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e") + if code != 0 || !strings.Contains(stdout, `"picked":[5]`) || !strings.Contains(stdout, `"missing":[]`) { + t.Errorf("run --defer 6 = %d, want #5 still picked:\n%s\n%s", code, stdout, stderr) + } +} + +// featWorld is #5071's shape: #3, a feat the line never gets, adds a file, +// and #5 fixes that file; #6 is a plain fix beside it. +func featWorld(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(5, "fix(app): fix the tap", "```release-note\nfix(app): the tap works.\n```\n", map[string]string{"app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f +} + +// A fix of a file a feat added on main, which the line never gets, is +// parked, not tried, and check takes the park as a note. +func TestParkAFixOfAFeatureTheLineLacks(t *testing.T) { + f := featWorld(t) + gh := &fakeGH{} + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + if strings.Contains(stdout, "### Deferred conflicts") || !strings.Contains(stdout, `"parked":[{"original":5,`) || !strings.Contains(stdout, `"kind":"dependency","needs":[3]`) { + t.Errorf("#5 is not parked on #3:\n%s", stdout) + } + if !strings.Contains(stdout, "### Parked") || !strings.Contains(stdout, "builds on [#3]") { + t.Errorf("the body does not say #5 builds on #3:\n%s", stdout) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "note: #5 is parked: it builds on #3, which never reaches release-1.1") { + t.Errorf("check = %d, want 0 with a note that #5 is parked:\n%s\n%s", code, stdout, stderr) + } +} + +// backport/include lifts no park. +func TestIncludeDoesNotLiftThePark(t *testing.T) { + f := featWorld(t) + code, stdout, stderr := f.exec(&fakeGH{labels: map[string][]int{labelInclude: {5}}}, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) || !strings.Contains(stdout, "`backport/include`, parked: builds on") { + t.Errorf("run with backport/include on #5 = %d, want #5 still parked:\n%s\n%s", code, stdout, stderr) + } +} + +// A prerequisite that is itself parked, but adapted onto the batch by hand, +// satisfies what builds on it: #3 is parked on #2, a feat, and #5 changes a +// line #3 wrote. +func TestAdaptedPrerequisiteSatisfiesItsDependent(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": appFile, "app/b.txt": appFile}) + f.lines() + f.pr(20, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + mc3 := f.pr(3, "fix(app): an excited two, a loud three", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": strings.Replace(appFile, "three", "THREE", 1)}) + f.pr(5, "fix(app): an excited three", "", map[string]string{"app/b.txt": strings.Replace(appFile, "three", "THREE!", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"kind":"dependency","needs":[3]`) { + t.Fatalf("run = %d, want #5 parked as long as #3 is:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + w := emptyBatch(f, &fakeGH{}, "batch\n\n"+deferredBlock(line11, nil, nil, nil)+"\n") + // #3 adapted by hand: every file it changes, changed, in the batch's + // shape. + pick := w.pick(w.head, f.git("rev-parse", mc3+"^2"), map[string]string{"app/a.txt": strings.Replace(appFile, "two", "two!", 1), "app/b.txt": strings.Replace(appFile, "three", "THREE", 1)}) + fold := w.merge(w.head, pick, "fix(app): an excited two, a loud three (#3)\n\n"+batchTrailer+": "+mc3) + f.git("push", "-q", "origin", fold+":refs/heads/"+batchBranch1) + w.gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, w.gh.prs[0].Body)} + code, stdout, stderr := f.exec(w.gh, "check", "--line", "release-1.1") + if code != 1 || strings.Contains(stderr, "#5 is parked") || strings.Contains(stdout+stderr, "waits on") || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check = %d, want #5 missing, not parked:\n%s\n%s", code, stdout, stderr) + } + code, stdout, stderr = f.exec(w.gh, "run", "--line", "release-1.1", "--continue") + if code != 0 || !strings.Contains(stdout, `"picked":[3,5]`) { + t.Errorf("run --continue = %d, want #5 picked after #3's adaptation:\n%s\n%s", code, stdout, stderr) + } +} + +// What an earlier unit of the batch brought counts: #3 writes the line #8 +// changes, a feat takes it back and another feat, #6, writes it again, so +// blame names #6, which never comes; once #3 is picked the branch has the +// line all the same. +func TestContentFromAnotherUnitSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(3, "fix(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(4, "feat(app): a quiet two", "", map[string]string{"app/a.txt": appFile}) + f.pr(6, "feat(app): a loud two again", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(8, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): a loud two (#3)", "fix(app): an excited two (#8)"}) { + t.Errorf("picked %q, want #3 and #8:\n%s", got, stdout) + } +} + +// auxWorld is #5142's shape: #3, a feat, adds a bats test; #5 fixes a +// workflow and that test, so all it builds on that never comes is a test; +// #6 is a plain fix. +func auxWorld(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n"}) + f.lines() + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(5, "fix(ci): run the release on tags", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f +} + +// A workflow original whose only missing dependency is a test waits for a +// maintainer in the scheduled run, whose token cannot push workflows, and +// check holds the cut on it. A maintainer's own run parks it. With no batch +// open, or one whose body carries no record, it is missing; a record from +// before the mode was saved says the scheduled mode by listing a wait. +func TestWorkflowModeCarriesToCheck(t *testing.T) { + f := auxWorld(t) + gh := &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `{"original":5,"files":[".github/workflows/release.yaml"],"workflows":true}`) { + t.Fatalf("scheduled run = %d, want #5 waiting, not parked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut = %d, want #5 waiting:\n%s\n%s", code, stdout, stderr) + } + v1 := "batch\n\n" + deferredBlock(line11, nil, nil, []pick{{v: &verdict{o: &original{number: 5}}, files: []string{".github/workflows/release.yaml"}}}) + "\n" + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, v1)} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut with a version 1 record = %d, want #5 waiting:\n%s\n%s", code, stdout, stderr) + } + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "a batch someone wrote")} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check under a batch with no record = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } + + f = auxWorld(t) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check with no batch = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } + gh = &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Fatalf("a maintainer's run = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked") { + t.Errorf("check after a maintainer's run = %d, want 0 with #5 parked:\n%s\n%s", code, stdout, stderr) + } +} + +// Evidence found only in a test file still parks an original that changes +// no workflow: #4973's shape. +func TestTestOnlyEvidenceParksOtherOriginals(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: nodes", "", map[string]string{"app/nodes.go": "package app\n", "app/nodes_test.go": "package app\n\nfunc a() {}\n"}) + f.lines() + f.pr(3, "feat(app): proxmox nodes", "", map[string]string{"app/nodes_test.go": "package app\n\nfunc a() { proxmox() }\n"}) + f.pr(5, "fix(app): nodes", "", map[string]string{"app/nodes.go": "package app\n\nvar fixed = true\n", "app/nodes_test.go": "package app\n\nfunc a() { proxmox(); fixed() }\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || strings.Contains(stdout, "### Deferred conflicts") || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked on #3:\n%s\n%s", code, stdout, stderr) + } +} + +// What an original builds on through lines it adds is not seen: #4973's +// $isProxmox. The original is picked; CI on the batch is what catches it. +func TestAddedLinesDependencyIsNotSeen(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: nodes", "", map[string]string{"app/nodes.tpl": "a\nb\n"}) + f.lines() + f.pr(3, "feat(app): define isProxmox", "", map[string]string{"app/proxmox.tpl": "isProxmox\n"}) + f.pr(5, "fix(app): use isProxmox", "", map[string]string{"app/nodes.tpl": "a\nif isProxmox\nb\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// With no batch open, check reads a closed record whatever its version and +// whatever mode it saved as the scheduled run's: the workflow original it +// left waiting is missing, not parked, while its drops and its other +// deferrals still count. +func TestClosedRecordModeFallback(t *testing.T) { + for _, version := range []int{2, 3} { + for _, saved := range []bool{false, true} { + t.Run(fmt.Sprintf("version %d, noWorkflowFiles %v", version, saved), func(t *testing.T) { + f := auxWorld(t) + mc6 := f.git("log", "--first-parent", "--format=%H", "-1", "--grep", "(#6)$", "main") + mc7 := f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: version, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: saved, Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 5, Files: []string{".github/workflows/release.yaml"}, Workflows: true}, {Original: 7, Commit: f.git("rev-parse", mc7+"^2"), Files: []string{"app/seven.txt"}}}, + Dropped: []droppedEntry{{Original: 6, Merge: mc6, Reason: "not for this line", On: 90, Date: "2026-10-26"}}, Stale: []int{}} + pr := f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n") + at := "2026-10-26T12:00:00Z" + pr.State, pr.MergedAt = "closed", &at + code, stdout, stderr := f.exec(&fakeGH{prs: []pullRequest{pr}}, "check", "--line", "release-1.1") + if code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 missing and not parked:\n%s\n%s", code, stdout, stderr) + } + if !strings.Contains(stderr, "note: #6 is dropped") || !strings.Contains(stderr, "warning: deferred conflict on #90 (merged): https://github.com/cozystack/cozystack/pull/7 ") { + t.Errorf("check lost the closed record's drop or deferral:\n%s", stderr) + } + }) + } + } +} + +// Withdrawing a drop puts the original back where the rules and the line +// put it: a partial original is parked again, in the record as in check. +func TestUndropRestoresThePark(t *testing.T) { + for name, world := range map[string]func(*testing.T) *fixture{ + "partial": func(t *testing.T) *fixture { + f, _ := policyWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f + }, + "dependency": featWorld, + } { + t.Run(name, func(t *testing.T) { + f := world(t) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "5", "--reason", "not for 1.1", "--push") + reopen(f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--undrop", "5", "--push") + reopen(f, gh) + if rec := recordOf(t, gh); len(rec.Parked) != 1 || rec.Parked[0].Original != 5 || rec.Parked[0].Kind != name { + t.Errorf("after the undrop the record parks %+v, want #5 parked %s", rec.Parked, name) + } + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked") { + t.Errorf("check = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + }) + } +} + +// A prerequisite the open batch carries is there for what builds on it, +// whatever its label now says: check and run agree that #7 is not parked. +func TestPrerequisiteStillOnTheBatchSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + gh := &fakeGH{} + openBatch(t, f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + gh.labels = map[string][]int{labelSkip: {5}} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#7 is parked") || !strings.Contains(stdout, "/pull/7 ") { + t.Errorf("check = %d, want #7 missing, not parked:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue"); code != 0 || !strings.Contains(stdout, `"picked":[7]`) { + t.Errorf("run --continue = %d, want #7 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// The picker judges what its picks did where the run has got to: #4 +// deletes the file #5 also deletes, so after #4 #5 would bring only its +// test, which the net merge at the line's head cannot see. +func TestPartialByAnEarlierPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.pr(4, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/four.txt": "4\n"}) + f.pr(40, "feat(app): the policy again", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"picked":[4]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #4 picked and #5 parked as partial:\n%s\n%s", code, stdout, stderr) + } +} + +// What an earlier pick brought clears a partial park the line's head +// gave: the line removed the policy, #4 brings it back, and #5's deletion +// then has something to delete. +func TestPartialClearedByAnEarlierPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(3, "feat(app): no policy", "", map[string]string{"app/policy.yaml": ""}) + f.pr(4, "fix(app): the policy, back", "", map[string]string{"app/policy.yaml": "allow: some\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"picked":[4,5]`) { + t.Errorf("run = %d, want #4 and #5 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// A reason a person deferred an original for comes before its partial +// park. +func TestReasonComesBeforeThePartialPark(t *testing.T) { + f, _ := policyWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "waits for the policy rework", "--push") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "warning: deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ") || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 deferred for its reason:\n%s\n%s", code, stdout, stderr) + } +} + +// The workflow exception holds only while what never comes is tests or +// docs, and while the workflow lines build on nothing missing: a +// workflow original that also fixes a file a feat added, or whose workflow +// line another original still to come wrote, parks in the scheduled run too. +func TestWorkflowExceptionNeedsTestsOnly(t *testing.T) { + t.Run("a never dependency outside tests", func(t *testing.T) { + f := auxWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(7, "fix(ci): test the tap, and fix it", "", map[string]string{".github/workflows/tap.yaml": "on: push\n", "app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":7,`) { + t.Errorf("scheduled run = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a workflow line still to come", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n", "app/four.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): four, the line's way", map[string]string{"app/four.txt": "line\n"}) + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(4, "fix(ci): four", "", map[string]string{"app/four.txt": "four\n", ".github/workflows/release.yaml": "on: [push]\njobs: {}\n"}) + f.pr(5, "fix(ci): run the release on tags", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("scheduled run = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + }) +} + +// What a rebuild takes off the batch is no longer there for what builds on +// it: deferring #5 makes #7, which merged after the batch was built, wait +// on it. +func TestRebuildTakesThePrerequisiteAway(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the e2e"); code != 0 || !strings.Contains(stdout, `{"original":7,"files":[],"needs":[5]}`) { + t.Errorf("run --defer 5 = %d, want #7 waiting on #5:\n%s\n%s", code, stdout, stderr) + } +} + +// A file the line renamed the way the original does is the original's own +// rename, not a deletion left without effect. +func TestSameRenameOnBothSidesIsWhole(t *testing.T) { + const long = "l01\nl02\nl03\nl04\nl05\nl06\nl07\nl08\nl09\nl10\nl11\nl12\n" + f := newFixture(t) + f.squash(2, "chore: a file", "", map[string]string{"app/p.txt": long}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames p to q", map[string]string{"app/p.txt": "", "app/q.txt": long}) + f.pr(5, "fix(app): rename p to q, and fix l10", "", map[string]string{"app/p.txt": "", "app/q.txt": strings.Replace(long, "l10", "L10", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked whole:\n%s\n%s", code, stdout, stderr) + } +} + +// A partial park only the picker saw, where the net merge at the line's +// head looks whole, holds in check and in the operations that pick nothing +// as the open record keeps it: check does not call the original missing. +func TestPickerParkHoldsInCheck(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.pr(4, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/four.txt": "4\n"}) + f.pr(40, "feat(app): the policy again", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked: only part of it applies") { + t.Errorf("check = %d, want #5 parked as the record says:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run --defer 6 = %d, want #5 still parked:\n%s\n%s", code, stdout, stderr) + } + // Deferring #4, which made #5 partial, ends the park: #5 applies whole + // on what the batch carries now. + f.git("switch", "-q", "main") + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "4", "--reason", "fails the e2e", "--push") + if code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"missing":[5]`) { + t.Errorf("run --defer 4 = %d, want #5 missing, no longer parked:\n%s\n%s", code, stdout, stderr) + } + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#5 is parked") || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check after deferring #4 = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } +} + +// A workflow original that waits for a maintainer only as the open record +// lists it: one merged since the batch was built is missing, and so is +// what waits on it. +func TestUnlistedWorkflowWaitIsMissing(t *testing.T) { + f := auxWorld(t) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + f.git("switch", "-q", "main") + reopen(f, gh) + f.pr(8, "fix(ci): run the new release on tags", "", map[string]string{".github/workflows/new.yaml": "on: tag\n", "hack/changelog.bats": "@test x {\n false\n}\n# eight\n", "app/eight.txt": appFile + "TWO\n"}) + f.pr(9, "fix(app): nine", "", map[string]string{"app/eight.txt": appFile + "TWO!\n"}) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/9 fix(app): nine (it waits on #8, which is not on the line either)") { + t.Errorf("check = %d, want #9 missing, waiting on #8:\n%s\n%s", code, stdout, stderr) + } +} + +// The line's own rename stands for the original's only with the original's +// change in it: a fold that leaves the renamed file as it was is refused. +func TestSameRenameStillNeedsItsFix(t *testing.T) { + const long = "l01\nl02\nl03\nl04\nl05\nl06\nl07\nl08\nl09\nl10\nl11\nl12\n" + f := newFixture(t) + f.squash(2, "chore: a file", "", map[string]string{"app/p.txt": long}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames p to q", map[string]string{"app/p.txt": "", "app/q.txt": long}) + mc5 := f.pr(5, "fix(app): rename p to q, fix l10, and five", "", map[string]string{"app/p.txt": "", "app/q.txt": strings.Replace(long, "l10", "L10", 1), "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): rename p to q, fix l10, and five (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/p.txt" without effect`) { + t.Errorf("fold without the fix in q = %d, %q\n%s", code, out, stderr) + } +} + +// A prerequisite the line carries is there, though the rules would keep it +// out: a feat carried by a hand adaptation does not park what builds on it. +func TestCarriedExcludedPrerequisiteSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + mc3 := f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "feat(app): a loud two, adapted\n\n(cherry picked from commit "+mc3+")", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "Two", 1)}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) { + t.Errorf("run = %d, want #5 not parked on the carried #3:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 missing, not parked:\n%s\n%s", code, stdout, stderr) + } +} + +// With no batch open, a workflow wait a closed record keeps waits on +// nothing: the original is judged as the scheduled run judges it, so one +// that also builds on a feat the line lacks is parked, in check as in run. +func TestClosedWorkflowWaitIsJudgedAsRunJudgesIt(t *testing.T) { + f := auxWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(7, "fix(ci): test the tap, and fix it", "", map[string]string{".github/workflows/tap.yaml": "on: push\n", "app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: 2, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: true, Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 7, Files: []string{".github/workflows/tap.yaml"}, Workflows: true}}, Dropped: []droppedEntry{}, Stale: []int{}} + pr := f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n") + at := "2026-10-26T12:00:00Z" + pr.State, pr.MergedAt = "closed", &at + gh := &fakeGH{prs: []pullRequest{pr}} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); strings.Contains(stdout, "/pull/7 ") || !strings.Contains(stderr, "note: #7 is parked: it builds on #20") { + t.Errorf("check --cut = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":7,`) { + t.Errorf("scheduled run = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } +} + +// A pick that comes out empty is judged too: #5's own commit did half its +// fix, which the line has, and a merge of main into its branch did the +// rest, which no pick carries; #5 is not on the line. +func TestEmptyPicksStillJudged(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: b", "", map[string]string{"app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): b, half of it", map[string]string{"app/b.txt": "middle\n"}) + f.git("switch", "-q", "-c", "pr-5", "main") + f.apply(map[string]string{"app/b.txt": "middle\n"}) + f.git("commit", "-q", "-m", "b, half of it") + f.git("switch", "-q", "main") + f.apply(map[string]string{"app/main.txt": "moved\n"}) + f.git("commit", "-q", "-m", "chore: move main along") + f.git("switch", "-q", "pr-5") + f.git("merge", "-q", "--no-ff", "--no-commit", "main") + f.apply(map[string]string{"app/b.txt": "fixed\n"}) + f.git("commit", "-q", "-m", "Merge branch 'main' into pr-5") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): b (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"on-line":[5]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked as partial, not on the line:\n%s\n%s", code, stdout, stderr) + } +} + +// --defer takes off a unit that does only part of its original whatever +// rule now keeps that original out, as check tells it to. +func TestDeferTakesOffAPartialExcludedUnit(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(chart): a network policy", "", map[string]string{"chart/templates/policy.yaml": "kind: NetworkPolicy\n"}) + mc5 := f.pr(5, "feat(chart): remove the policy, and test values", "", map[string]string{"chart/templates/policy.yaml": "", "chart/tests/values_test.yaml": "suite: values\n"}) + gh := &fakeGH{} + handBatch(f, gh, mc5) + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "only part of it applies") { + t.Fatalf("check --cut = %d, want the partial unit to fail it:\n%s", code, stdout) + } + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "only part of it applies", "--push"); code != 0 || prState(gh.find(90)) != "closed" { + t.Errorf("run --defer 5 --push = %d, #90 %s; want the unit taken off and the batch closed:\n%s", code, prState(gh.find(90)), stderr) + } +} + +// A stored wait on an original merged after the one that waits cannot be +// what a run wrote. +func TestForwardWaitIsRefused(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: recordVersion, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 5, Files: []string{}, Needs: []int{7}}}, Dropped: []droppedEntry{}, Stale: []int{}} + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+batchBranch1) + gh := &fakeGH{prs: []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n")}} + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "#5 wait on #7") { + t.Errorf("check = %d, want 2 for a wait on a later original:\n%s", code, stderr) + } +} + +// The workflow exception looks only at what still keeps the original from +// the line: a workflow line its prerequisite wrote, which the open batch +// carries though that prerequisite is labelled backport/skip, does not +// stop it, in check as in run. +func TestWorkflowExceptionIgnoresWhatIsThere(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n"}) + f.lines() + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(4, "fix(ci): run the release on pushes", "", map[string]string{".github/workflows/release.yaml": "on: [push]\njobs: {}\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(ci): run the release on tags too", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + f.git("switch", "-q", "main") + reopen(f, gh) + gh.labels = map[string][]int{labelSkip: {4}} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push"); code != 0 || !strings.Contains(stdout, `{"original":7,"files":[".github/workflows/release.yaml"],"workflows":true}`) { + t.Fatalf("scheduled run = %d, want #7 waiting for a maintainer:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#7 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut = %d, want #7 waiting, as run has it:\n%s\n%s", code, stdout, stderr) + } +} + +// An empty pick leaves a deletion undone that only a merge of main into the +// original's branch made: the original is not on the line. +func TestEmptyPickLeavesADeletionUndone(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old feature\n", "app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): b, already", map[string]string{"app/b.txt": "fixed\n"}) + f.git("switch", "-q", "-c", "pr-5", "main") + f.apply(map[string]string{"app/b.txt": "fixed\n"}) + f.git("commit", "-q", "-m", "b") + f.git("switch", "-q", "main") + f.apply(map[string]string{"app/main.txt": "moved\n"}) + f.git("commit", "-q", "-m", "chore: move main along") + f.git("switch", "-q", "pr-5") + f.git("merge", "-q", "--no-ff", "--no-commit", "main") + f.git("rm", "-q", "app/a.txt") + f.git("commit", "-q", "-m", "Merge branch 'main' into pr-5") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): b, and a goes (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"on-line":[5]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked as partial, not on the line:\n%s\n%s", code, stdout, stderr) + } +} + +// A file the line already has as the original makes it builds on nothing +// the line lacks: #5 fixes a.txt, which the line fixed the same way, and +// b.txt, so it picks whole and is not parked on the feat that wrote a.txt. +func TestAlreadyAppliedFileBuildsOnNothing(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": appFile, "app/b.txt": "old\n"}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "fix(app): two, the line's way", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + f.pr(5, "fix(app): an excited two, and b", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": "fixed\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked, not parked:\n%s\n%s", code, stdout, stderr) + } +} diff --git a/cmd/backport-batch/record.go b/cmd/backport-batch/record.go new file mode 100644 index 000000000..37dd7946a --- /dev/null +++ b/cmd/backport-batch/record.go @@ -0,0 +1,682 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "regexp" + "slices" + "strings" + "time" +) + +// A line's record is the backport-batch block of its newest batch pull +// request, read whatever that pull request's state and written only while it +// is open: a merged or closed pull request is never written again, so what it +// recorded stays what the line's later commands act on until a newer batch +// carries a record of its own. + +// lineRecord is the one line of JSON in the block. Version 1, the block +// before it had a version, holds only line and deferred; version 3 adds what +// is parked and which units on the batch do only part of their original. +type lineRecord struct { + Version int `json:"version"` + Line string `json:"line"` + Built *builtFrom `json:"built,omitempty"` + Report string `json:"report,omitempty"` + Closing *closing `json:"closing"` + Deferred []deferredEntry `json:"deferred"` + Dropped []droppedEntry `json:"dropped"` + Stale []int `json:"stale"` + Parked []parkedEntry `json:"parked,omitempty"` + Partial []int `json:"partial,omitempty"` + // Changed are the picks that differ from their originals. + Changed []int `json:"changed,omitempty"` +} + +// recordVersion is the version a run writes. +const recordVersion = 3 + +// lineRecordV2 is version 2, which has none of version 3's keys. +type lineRecordV2 struct { + Version int `json:"version"` + Line string `json:"line"` + Built *builtFrom `json:"built,omitempty"` + Report string `json:"report,omitempty"` + Closing *closing `json:"closing"` + Deferred []deferredEntry `json:"deferred"` + Dropped []droppedEntry `json:"dropped"` + Stale []int `json:"stale"` +} + +// The kinds of park: an original that builds on what never reaches the +// line, and one whose clean pick would leave part of it without effect. +const ( + parkDependency = "dependency" + parkPartial = "partial" +) + +// parkedEntry is an original the batch leaves out because it cannot land on +// the line whole: what it needs that never comes, or the paths it would +// leave without effect. announced says notify has named it on the open +// batch, for exactly this entry. +type parkedEntry struct { + Original int `json:"original"` + Merge string `json:"merge"` + Kind string `json:"kind"` + Needs []int `json:"needs,omitempty"` + Commits []string `json:"commits,omitempty"` + Files []string `json:"files,omitempty"` + Announced bool `json:"announced,omitempty"` +} + +// parkEvent is what makes a park the same park from one record to the +// next: its merge, kind, prerequisites, commits and files. A park whose +// event changed is news, and notify names it again. +func parkEvent(p parkedEntry) string { + data, _ := json.Marshal(struct { + Merge string `json:"merge"` + Kind string `json:"kind"` + Needs []int `json:"needs"` + Commits []string `json:"commits"` + Files []string `json:"files"` + }{p.Merge, p.Kind, p.Needs, p.Commits, p.Files}) + return string(data) +} + +// carryAnnounced marks each park announced that the carrier's record holds +// announced as the same event, a closed carrier's too, so a new batch does +// not name again what an earlier one named. +func carryAnnounced(parked []parkedEntry, carrier *lineRecord) []parkedEntry { + out := slices.Clone(parked) + for i := range out { + out[i].Announced = false + } + if carrier == nil { + return out + } + for i := range out { + out[i].Announced = slices.ContainsFunc(carrier.Parked, func(c parkedEntry) bool { + return c.Announced && c.Original == out[i].Original && parkEvent(c) == parkEvent(out[i]) + }) + } + return out +} + +// validate holds a parked entry to what a run writes. Its errors read after +// "the block" or "report ". +func (p parkedEntry) validate() error { + sortedUnique := func(n int, less func(i int) bool) bool { + for i := 1; i < n; i++ { + if !less(i) { + return false + } + } + return true + } + switch { + case checkSHA(p.Merge) != nil: + return fmt.Errorf("parks #%d by merge %q", p.Original, p.Merge) + case !sortedUnique(len(p.Needs), func(i int) bool { return p.Needs[i-1] < p.Needs[i] }) || slices.ContainsFunc(p.Needs, func(n int) bool { return n <= 0 || n == p.Original }): + return fmt.Errorf("parks #%d for originals %v, which are not what a run writes", p.Original, p.Needs) + case !sortedUnique(len(p.Commits), func(i int) bool { return p.Commits[i-1] < p.Commits[i] }) || slices.ContainsFunc(p.Commits, func(c string) bool { return checkSHA(c) != nil }): + return fmt.Errorf("parks #%d for commits %q, which are not what a run writes", p.Original, p.Commits) + case !sortedUnique(len(p.Files), func(i int) bool { return p.Files[i-1] < p.Files[i] }) || !slices.Equal(p.Files, cleanFiles(p.Files)): + return fmt.Errorf("parks #%d for files %q, which are not what a run writes", p.Original, p.Files) + } + switch p.Kind { + case parkPartial: + if len(p.Files) == 0 || len(p.Needs)+len(p.Commits) > 0 { + return fmt.Errorf("parks #%d as partial with no files it leaves without effect, or with what it needs", p.Original) + } + case parkDependency: + if len(p.Needs)+len(p.Commits) == 0 || len(p.Files) > 0 { + return fmt.Errorf("parks #%d for a dependency it does not name, or with files", p.Original) + } + default: + return fmt.Errorf("parks #%d as %q, which is no kind of park", p.Original, p.Kind) + } + return nil +} + +// builtFrom is what a classification was made of; batch is the head it +// describes. +type builtFrom struct { + Main string `json:"main"` + Line string `json:"line"` + Newer string `json:"newer"` + Policy string `json:"policy"` + Batch string `json:"batch"` + Tool string `json:"tool"` + Run string `json:"run"` + At string `json:"at"` + Options runOptions `json:"options"` +} + +type runOptions struct { + NoWorkflowFiles bool `json:"noWorkflowFiles"` + Mode string `json:"mode"` +} + +// The operations a record is written by: a run that assembles the batch, and +// the ones that only change what the record says about it. +const ( + modeNew = "new" + modeContinue = "continue" + modeDefer = "defer" + modeDrop = "drop" + modeUndrop = "undrop" + modeClose = "close" +) + +var modes = []string{modeNew, modeContinue, modeDefer, modeDrop, modeUndrop, modeClose} + +// closing is written into the record of an open batch that the run is about +// to close: the head it froze, which the close and the branch's deletion are +// leased on. +type closing struct { + PR int `json:"pr"` + Branch string `json:"branch"` + Head string `json:"head"` +} + +// droppedEntry is a maintainer's decision that an original stays off the +// line, bound to its merge commit on main. +type droppedEntry struct { + Original int `json:"original"` + Merge string `json:"merge"` + Reason string `json:"reason"` + On int `json:"on"` + Date string `json:"date"` +} + +var ( + toolRE = regexp.MustCompile(`^[0-9a-f]{40}(\+dirty)?$`) + reportIDRE = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// oneLineText is s as a run writes a reason: one line, cleaned and capped. +// A record holding anything else was not written by a run, and its text +// would reach a terminal, a body or a comment as someone else wrote it. +func oneLineText(s string) string { return capped(oneLine(clean(s, false)), titleLimit) } + +// fencedLines lists the one line of each block that fence opens. +func fencedLines(body, fence string) ([]string, error) { + var blocks []string + lines := strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") + for i := 0; i < len(lines); i++ { + if strings.TrimSpace(lines[i]) != fence { + continue + } + if i+2 >= len(lines) || strings.TrimSpace(lines[i+2]) != "```" { + return nil, fmt.Errorf("the %s block is not one line of JSON", fence) + } + blocks = append(blocks, lines[i+1]) + i += 2 + } + return blocks, nil +} + +// decodeStrict reads exactly one JSON value into v, refusing a key v does not +// have and anything after the value. +func decodeStrict(data string, v any) error { + dec := json.NewDecoder(strings.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + var rest json.RawMessage + if dec.More() || dec.Decode(&rest) != io.EOF { + return errors.New("something follows the JSON value") + } + return nil +} + +// parseRecord reads the line's record from a batch body: nil when the body +// has no block, an error when it has more than one or one it cannot vouch +// for whole. +func parseRecord(body string, l line) (*lineRecord, error) { + blocks, err := fencedLines(body, deferredFence) + switch { + case err != nil: + return nil, err + case len(blocks) == 0: + return nil, nil + case len(blocks) > 1: + return nil, fmt.Errorf("the body holds %d %s blocks", len(blocks), deferredFence) + } + var head struct { + Version *int `json:"version"` + } + if err := json.Unmarshal([]byte(blocks[0]), &head); err != nil { + return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) + } + version := 1 + if head.Version != nil { + version = *head.Version + } + var rec lineRecord + switch version { + case 1: + var v1 struct { + Version int `json:"version"` + Line string `json:"line"` + Deferred []deferredEntry `json:"deferred"` + } + err = decodeStrict(blocks[0], &v1) + rec = lineRecord{Version: 1, Line: v1.Line, Deferred: v1.Deferred} + case 2: + var v2 lineRecordV2 + err = decodeStrict(blocks[0], &v2) + rec = lineRecordWithout(v2) + case 3: + err = decodeStrict(blocks[0], &rec) + default: + return nil, fmt.Errorf("the %s block is version %d, which this backport-batch does not read; it reads 1 to %d", deferredFence, version, recordVersion) + } + if err == nil && version < 3 && slices.ContainsFunc(rec.Deferred, func(d deferredEntry) bool { return d.Needs != nil }) { + err = fmt.Errorf("version %d has no waits on other originals", version) + } + if err != nil { + return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) + } + if err := rec.validate(l); err != nil { + return nil, fmt.Errorf("the %s block %w", deferredFence, err) + } + return &rec, nil +} + +// validate holds the record to what a run writes. Its errors read after +// "the block". +func (rec *lineRecord) validate(l line) error { + if rec.Line != l.branch() { + return fmt.Errorf("is for %q, not %s", rec.Line, l.branch()) + } + seen := map[int]bool{} + once := func(n int) error { + if n <= 0 { + return fmt.Errorf("names original %d", n) + } + if seen[n] { + return fmt.Errorf("names #%d twice", n) + } + seen[n] = true + return nil + } + for _, d := range rec.Deferred { + if err := once(d.Original); err != nil { + return err + } + if d.Commit != "" && checkSHA(d.Commit) != nil { + return fmt.Errorf("names commit %q for #%d", d.Commit, d.Original) + } + // A conflict names the commit that failed and its files, a deferral + // its reason, and a wait its workflow files. + kinds := 0 + for _, is := range []bool{d.Commit != "" || len(d.Files) > 0 && !d.Workflows && d.Reason == "" && len(d.Needs) == 0, d.Reason != "", d.Workflows, len(d.Needs) > 0} { + if is { + kinds++ + } + } + switch { + case d.Workflows && (d.Commit != "" || d.Reason != ""): + return fmt.Errorf("has #%d wait for a maintainer and defers it as well", d.Original) + case kinds != 1: + return fmt.Errorf("does not say whether #%d is a conflict, deferred for a reason or waiting", d.Original) + case d.Reason != "" && strings.TrimSpace(d.Reason) == "": + return fmt.Errorf("defers #%d for an empty reason", d.Original) + case d.Reason != oneLineText(d.Reason): + return fmt.Errorf("defers #%d for %q, which is no reason a run writes", d.Original, d.Reason) + case !slices.Equal(d.Files, cleanFiles(d.Files)): + return fmt.Errorf("names files of #%d that are not as a run writes them: %q", d.Original, d.Files) + case d.More < 0: + return fmt.Errorf("counts %d more files for #%d", d.More, d.Original) + case len(d.Needs) > 0 && (len(d.Files) > 0 || d.More != 0): + return fmt.Errorf("has #%d wait on other originals with files of its own", d.Original) + } + for i, n := range d.Needs { + if n <= 0 || n == d.Original || i > 0 && d.Needs[i-1] >= n { + return fmt.Errorf("has #%d wait on %v, which is not what a run writes", d.Original, d.Needs) + } + } + } + for _, d := range rec.Dropped { + if err := once(d.Original); err != nil { + return err + } + if _, err := time.Parse(time.DateOnly, d.Date); err != nil || checkSHA(d.Merge) != nil || strings.TrimSpace(d.Reason) == "" || d.On <= 0 { + return fmt.Errorf("drops #%d with a merge, reason, pull request or date it cannot have", d.Original) + } + if d.Reason != oneLineText(d.Reason) { + return fmt.Errorf("drops #%d for %q, which is no reason a run writes", d.Original, d.Reason) + } + } + for _, p := range rec.Parked { + if err := once(p.Original); err != nil { + return err + } + if err := p.validate(); err != nil { + return err + } + } + staleSeen := map[int]bool{} + for _, n := range rec.Stale { + if n <= 0 || staleSeen[n] { + return fmt.Errorf("lists %d as still on the batch more than once or not at all", n) + } + staleSeen[n] = true + } + changedSeen := map[int]bool{} + for _, n := range rec.Changed { + if n <= 0 || changedSeen[n] { + return fmt.Errorf("lists %d as a changed pick more than once or not at all", n) + } + changedSeen[n] = true + } + if err := rec.waitsHold(); err != nil { + return err + } + // What is on the batch is neither deferred nor parked. + left := map[int]bool{} + for _, d := range rec.Deferred { + left[d.Original] = true + } + for _, p := range rec.Parked { + left[p.Original] = true + if slices.Contains(p.Commits, p.Merge) { + return fmt.Errorf("parks #%d for its own merge", p.Original) + } + } + for _, list := range [][]int{rec.Partial, rec.Changed} { + for _, n := range list { + if left[n] { + return fmt.Errorf("has #%d on the batch and left off it at once", n) + } + } + } + partialSeen := map[int]bool{} + for _, n := range rec.Partial { + // A unit that does only part of its original is still on the batch. + if !staleSeen[n] || partialSeen[n] { + return fmt.Errorf("lists #%d as a partial unit more than once, or not as still on the batch", n) + } + partialSeen[n] = true + } + if rec.Version < 2 { + return nil + } + if b := rec.Built; b == nil { + return errors.New("does not say what it was built from") + } else if err := b.validate(); err != nil { + return err + } + if !reportIDRE.MatchString(rec.Report) { + return fmt.Errorf("names report %q", rec.Report) + } + if c := rec.Closing; c != nil { + m := batchBranchRE.FindStringSubmatch(c.Branch) + if c.PR <= 0 || m == nil || m[1] != l.branch() || checkSHA(c.Head) != nil { + return fmt.Errorf("closes #%d on %q at %q, which is no batch of %s", c.PR, c.Branch, c.Head, l.branch()) + } + } + return nil +} + +func (b *builtFrom) validate() error { + for _, sha := range []string{b.Main, b.Line, b.Policy, b.Batch} { + if checkSHA(sha) != nil { + return fmt.Errorf("says it was built from %q", sha) + } + } + if b.Newer != "" && checkSHA(b.Newer) != nil { + return fmt.Errorf("says it was built from %q", b.Newer) + } + if b.Tool != "" && !toolRE.MatchString(b.Tool) { + return fmt.Errorf("names tool %q", b.Tool) + } + if b.Run != "" && !runURLRE.MatchString(b.Run) { + return fmt.Errorf("names run %q", b.Run) + } + if _, err := time.Parse(time.RFC3339, b.At); err != nil { + return fmt.Errorf("was built at %q", b.At) + } + if !slices.Contains(modes, b.Options.Mode) { + return fmt.Errorf("was written by %q, which is no operation of backport-batch", b.Options.Mode) + } + return nil +} + +var runURLRE = regexp.MustCompile(`^https://[^/\s]+/[^/\s]+/[^/\s]+/actions/runs/[0-9]+(/attempts/[0-9]+)?$`) + +// waitsHold holds what waits and parks need to what the record says of the +// originals they need: a wait needs ones still to come, so none parked or +// dropped, a park ones that never come, so none deferred, and no wait ends +// up waiting on itself. Its errors read after "the block". +func (rec *lineRecord) waitsHold() error { + deferred, never := map[int][]int{}, map[int]bool{} + for _, d := range rec.Deferred { + deferred[d.Original] = d.Needs + } + for _, p := range rec.Parked { + never[p.Original] = true + } + for _, d := range rec.Dropped { + never[d.Original] = true + } + for _, d := range rec.Deferred { + for _, n := range d.Needs { + if never[n] { + return fmt.Errorf("has #%d wait on #%d, which never comes", d.Original, n) + } + } + } + for _, p := range rec.Parked { + for _, n := range p.Needs { + if _, ok := deferred[n]; ok { + return fmt.Errorf("parks #%d for #%d, which is still to come", p.Original, n) + } + } + } + needs := map[int][]int{} + for _, d := range rec.Deferred { + needs[d.Original] = d.Needs + } + for _, p := range rec.Parked { + needs[p.Original] = p.Needs + } + return waitCycle(needs) +} + +// waitCycle refuses waits that lead back to where they started, which no +// run writes: a prerequisite merged before what needs it. +func waitCycle(needs map[int][]int) error { + state := map[int]int{} // 1 on the path, 2 done + var visit func(n int) error + visit = func(n int) error { + switch state[n] { + case 1: + return fmt.Errorf("has #%d wait on itself through what it waits on", n) + case 2: + return nil + } + state[n] = 1 + for _, f := range needs[n] { + if err := visit(f); err != nil { + return err + } + } + state[n] = 2 + return nil + } + for _, n := range slices.Sorted(maps.Keys(needs)) { + if err := visit(n); err != nil { + return err + } + } + return nil +} + +// lineRecordWithout is a version 2 record as the newest version holds it. +func lineRecordWithout(v2 lineRecordV2) lineRecord { + return lineRecord{Version: v2.Version, Line: v2.Line, Built: v2.Built, Report: v2.Report, Closing: v2.Closing, Deferred: v2.Deferred, Dropped: v2.Dropped, Stale: v2.Stale} +} + +// recordBlock is the block that holds rec. One line of JSON: encoding/json +// escapes newlines and <, > and &, so nothing in a file name or a reason can +// end the fence or open HTML. +func recordBlock(rec lineRecord) string { + data, _ := json.Marshal(rec) + return deferredFence + "\n" + string(data) + "\n```" +} + +// savedNoWorkflowFiles is the --no-workflow-files the open batch was last +// assembled with, which an operation that only changes the record keeps. A +// record from before it was saved says so by listing a wait. +func (s recordSource) savedNoWorkflowFiles() bool { + if !s.open || s.rec == nil { + return false + } + if s.rec.Built != nil { + return s.rec.Built.Options.NoWorkflowFiles + } + return slices.ContainsFunc(s.rec.Deferred, func(d deferredEntry) bool { return d.Workflows }) +} + +// dropped is what the record drops, which every later record keeps. +func (rec *lineRecord) dropped() []droppedEntry { + if rec == nil { + return nil + } + return slices.Clone(rec.Dropped) +} + +// deferred maps the originals the record defers to their entries. +func (rec *lineRecord) deferred() map[int]deferredEntry { + out := map[int]deferredEntry{} + if rec != nil { + for _, d := range rec.Deferred { + out[d.Original] = d + } + } + return out +} + +// recordSource is where a line's record was found. +type recordSource struct { + // carrier is the pull request whose block is the record, nil when the + // line has none. + carrier *pullRequest + // open says the carrier is the open batch, so the record may be written. + open bool + rec *lineRecord + // code is the open batch pull request, if any. One whose body someone + // wrote by hand is the batch the line builds on but carries no record. + code *pullRequest +} + +// closingOpen is the close the record says is under way on a batch that is +// still open, or nil. +func (s recordSource) closingOpen() *closing { + if !s.open || s.rec == nil { + return nil + } + return s.rec.Closing +} + +func prState(pr *pullRequest) string { + switch { + case pr.MergedAt != nil: + return "merged" + case pr.State == "" || pr.State == "open": + return "open" + case pr.State == "closed": + return "closed" + } + return fmt.Sprintf("in state %q", pr.State) +} + +func hasRecord(body string) bool { + return slices.ContainsFunc(strings.Split(body, "\n"), func(ln string) bool { return strings.TrimSpace(ln) == deferredFence }) +} + +// findLineRecord finds the line's record: the block of the newest pull +// request opened from a batch branch of this repository for the line by an +// author the batch trusts, open, merged or closed. open is the open batch +// pull request batchPull chose, or nil. Anything that leaves in doubt which +// record is the newest is an error, never a fall back to an older one: a +// newer block by an author the batch does not trust, a block the bot wrote +// and someone removed, a record it cannot read, a closed record newer than +// the open batch, or more pull requests than it reads. +func findLineRecord(gh github, l line, open *pullRequest) (recordSource, error) { + src := recordSource{code: open} + for page := 1; src.carrier == nil; page++ { + if page > recentPullsPages { + return src, fmt.Errorf("none of the %d newest pull requests against %s is a batch pull request with a record, so the line's record cannot be told apart from none", recentPullsPages*recentPullsPage, l.branch()) + } + prs, err := gh.recentPulls(l.branch(), page) + if err != nil { + return src, err + } + for i := range prs { + pr := &prs[i] + m := batchBranchRE.FindStringSubmatch(pr.Head.Ref) + if m == nil || m[1] != l.branch() || pr.Base.Ref != l.branch() || pr.Head.Repo == nil || pr.Head.Repo.FullName != pr.Base.Repo.FullName { + continue + } + if open != nil && pr.Number == open.Number { + pr = open + } + if !hasRecord(pr.Body) { + if pr.User.Type == "Bot" && gh.batchBot() != "" && pr.User.Login == gh.batchBot() { + return src, fmt.Errorf("batch pull request #%d (%s) was opened by %s but carries no %s block; someone removed it, so the line's record is unknown", pr.Number, prState(pr), pr.User.Login, deferredFence) + } + continue + } + why, err := trusted(gh, *pr) + if err != nil { + return src, fmt.Errorf("batch pull request #%d: %w", pr.Number, err) + } + if why != "" { + return src, fmt.Errorf("batch pull request #%d (%s) carries a %s block but was %s; an older record is not taken in its place", pr.Number, prState(pr), deferredFence, why) + } + rec, err := parseRecord(pr.Body, l) + if err != nil { + return src, fmt.Errorf("batch pull request #%d (%s): %w", pr.Number, prState(pr), err) + } + src.carrier, src.rec = pr, rec + break + } + if len(prs) < recentPullsPage { + break + } + } + switch c := src.carrier; { + case open != nil && hasRecord(open.Body) && (c == nil || c.Number < open.Number): + return src, fmt.Errorf("GitHub does not list the open batch pull request #%d among the pull requests against %s; run again", open.Number, l.branch()) + case open != nil && hasRecord(open.Body) && c.Number > open.Number: + return src, fmt.Errorf("batch pull request #%d (%s) carries a record newer than the open batch pull request #%d; close one of them by hand", c.Number, prState(c), open.Number) + case open != nil && c == open: + src.open = true + case c != nil && prState(c) == "open": + return src, fmt.Errorf("batch pull request #%d carries the line's record and is open, but is not the batch this run builds on", c.Number) + } + return src, nil +} diff --git a/cmd/backport-batch/record_test.go b/cmd/backport-batch/record_test.go new file mode 100644 index 000000000..4bbddc58c --- /dev/null +++ b/cmd/backport-batch/record_test.go @@ -0,0 +1,433 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// The records the live batches carried before records had a version, as +// GitHub returned them on 2026-11-09, read as version 1. +func TestRecordReadsVersion1(t *testing.T) { + for line, block := range map[string]string{ + "release-1.7": `{"line":"release-1.7","deferred":[{"original":4756,"commit":"647407545d630291cfe2a20880cce45af22edf0c","files":["packages/system/backupstrategy-controller/values.yaml"]},{"original":4867,"commit":"813c6180861efe1896c26d62a7afcbc1ff590ebc","files":[".github/renovate.json"]},{"original":5082,"commit":"e93db8848027450c2e23d21348409efa641cb6f1","files":["pkg/registry/core/tap/rest.go"]},{"original":4888,"files":[".github/workflows/cut-prerelease.yaml"],"workflows":true},{"original":4971,"files":[".github/workflows/e2e-proxmox.yaml"],"workflows":true}]}`, + "release-1.6": `{"line":"release-1.6","deferred":[{"original":5071,"commit":"3fa534f3cf6cb5774a183f0202f7f5c13e9ae211","files":["internal/operator/tapmaterializer_reconciler.go","internal/operator/tapmaterializer_reconciler_test.go"]},{"original":5086,"commit":"5265ee662c591db409c8ef1ed8bb457cf29f5c22","files":["packages/apps/tenant/templates/gateway.yaml"]},{"original":5087,"commit":"8d8179f20a2c10de03face1472435d5083cde38d","files":["internal/controller/applicationdefinition_helmreconciler.go","internal/controller/applicationdefinition_helmreconciler_test.go"]},{"original":5142,"files":[".github/workflows/pull-requests-release.yaml",".github/workflows/update-releasenotes.yaml"],"workflows":true}]}`, + } { + l, _ := parseLine(line) + rec, err := parseRecord("## What this PR does\n\ntext\n\n"+deferredFence+"\n"+block+"\n```\n\n### Release note\n", l) + if err != nil { + t.Fatalf("%s: parseRecord = %v", line, err) + } + waits, conflicts := 0, 0 + for _, e := range rec.Deferred { + switch { + case e.Workflows: + waits++ + case e.Commit != "": + conflicts++ + } + } + if rec.Version != 1 || rec.Line != line || rec.Closing != nil || len(rec.Dropped) != 0 || waits+conflicts != len(rec.Deferred) || waits == 0 || conflicts == 0 { + t.Errorf("%s: record = %+v; want version 1 with its conflicts and waits", line, rec) + } + } +} + +// v2 is a version 2 record with each value it may hold. +func v2(extra string) string { + sha := strings.Repeat("a", 40) + return `{"version":2,"line":"release-1.1","built":{"main":"` + sha + `","line":"` + sha + `","newer":"","policy":"` + sha + `","batch":"` + sha + `","tool":"` + sha + `+dirty","run":"","at":"2026-10-26T06:00:00Z","options":{"noWorkflowFiles":true,"mode":"continue"}},` + + `"report":"` + strings.Repeat("b", 64) + `","closing":null,"deferred":[{"original":5,"commit":"` + sha + `","files":["app/five.txt"]},{"original":6,"files":[],"reason":"fails the e2e"},{"original":7,"files":[".github/workflows/x.yaml"],"workflows":true}],` + + `"dropped":[{"original":8,"merge":"` + sha + `","reason":"needs #4","on":90,"date":"2026-10-26"}],"stale":[9]` + extra + `}` +} + +// A record is read whole or not at all: anything it cannot vouch for stops +// the command rather than being half believed. +func TestRecordRefuses(t *testing.T) { + l, _ := parseLine("release-1.1") + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + if rec, err := parseRecord(block(v2("")), l); err != nil || rec.Version != 2 || len(rec.Dropped) != 1 || rec.Built.Options.Mode != "continue" { + t.Fatalf("parseRecord of a whole v2 record = %+v, %v", rec, err) + } + sha := strings.Repeat("a", 40) + for name, json := range map[string]string{ + "version 4": strings.Replace(v2(""), `"version":2`, `"version":4`, 1), + "trailing data": v2("") + `{}`, + "trailing text": v2("") + ` x`, + "an original listed twice": strings.Replace(v2(""), `"original":8`, `"original":5`, 1), + "a needs entry with a commit": strings.Replace(v2(""), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"commit":"`+sha+`","files":[],"needs":[4]}`, 1), + "an empty reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":""`, 1), + "an empty drop reason": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":" "`, 1), + "a bad tool": strings.Replace(v2(""), `+dirty"`, `-dirty"`, 1), + "an unknown mode": strings.Replace(v2(""), `"mode":"continue"`, `"mode":"rebase"`, 1), + "a reason with a commit": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"x","commit":"`+sha+`"`, 1), + "a short merge": strings.Replace(v2(""), `"merge":"`+sha+`"`, `"merge":"abc"`, 1), + "a bad date": strings.Replace(v2(""), `"date":"2026-10-26"`, `"date":"26.10.2026"`, 1), + "a bad time": strings.Replace(v2(""), `"at":"2026-10-26T06:00:00Z"`, `"at":"yesterday"`, 1), + "a short report": strings.Replace(v2(""), strings.Repeat("b", 64), "bbbb", 1), + "a closing on another line": strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"backport-batch/release-1.0-20261026","head":"`+sha+`"}`, 1), + "a closing without a head": strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"`+batchBranch1+`","head":""}`, 1), + "a negative stale": strings.Replace(v2(""), `"stale":[9]`, `"stale":[-9]`, 1), + "an unknown key": v2(`,"parked":[]`), + "an entry of no kind": strings.Replace(v2(""), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"files":[]}`, 1), + "another line": strings.Replace(v2(""), `"line":"release-1.1"`, `"line":"release-1.0"`, 1), + "an escape in a reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"fails\u001b[2J the e2e"`, 1), + "a newline in a reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"fails\nwarning: forged"`, 1), + "an escape in a drop": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"needs\u001b]0;x\u0007 #4"`, 1), + "a newline in a drop": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"needs #4\nnote: forged"`, 1), + "a reason past its length": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"`+strings.Repeat("x", titleLimit+1)+`"`, 1), + "a bidi file name": strings.Replace(v2(""), `"files":["app/five.txt"]`, `"files":["app/\u202efive.txt"]`, 1), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } + closing := strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"`+batchBranch1+`","head":"`+sha+`"}`, 1) + if rec, err := parseRecord(block(closing), l); err != nil || rec.Closing == nil || rec.Closing.PR != 90 { + t.Errorf("a record that is closing = %+v, %v", rec, err) + } +} + +// recordPR is a batch pull request on release-1.1 carrying body, numbered n, +// in state, opened by login. +func recordPR(n int, state, login, body string) pullRequest { + pr := notifyPR("20261026", body) + pr.Number, pr.State = n, state + pr.Head.Ref = fmt.Sprintf("backport-batch/release-1.1-202610%02d", n%30+1) + pr.User.Login = login + if strings.HasSuffix(login, "[bot]") { + pr.User.Type = "Bot" + } + return pr +} + +func recordBody(conflict int) string { + return "text\n\n" + deferredFence + "\n" + fmt.Sprintf(`{"line":"release-1.1","deferred":[{"original":%d,"commit":"%s","files":["a"]}]}`, conflict, strings.Repeat("a", 40)) + "\n```\n" +} + +// The record is the block of the newest batch pull request that carries one, +// whatever its state, among those a trusted author opened from a batch branch +// of this repository; anything that leaves the newest record in doubt stops +// the command rather than falling back to an older one. +func TestFindLineRecord(t *testing.T) { + const bot = defaultBatchBot + other := func(n int) pullRequest { + pr := recordPR(n, "closed", "maintainer", "") + pr.Head.Ref = fmt.Sprintf("fix-%d", n) + return pr + } + fork := recordPR(92, "closed", "maintainer", recordBody(92)) + fork.Head.Repo = &struct { + FullName string `json:"full_name"` + }{"someone/cozystack"} + cases := []struct { + name string + prs []pullRequest + open int // the open batch the caller found, or 0 + bot string + carrier int // 0 for none + isOpen bool + refusal string + pages int + }{ + {name: "the open batch first", prs: []pullRequest{recordPR(92, "open", "maintainer", recordBody(92)), recordPR(91, "closed", "maintainer", recordBody(91))}, open: 92, carrier: 92, isOpen: true}, + {name: "the newest closed record", prs: []pullRequest{recordPR(91, "closed", bot, recordBody(91)), recordPR(90, "closed", bot, recordBody(90))}, bot: bot, carrier: 91}, + {name: "a person's batch without a block is passed over", prs: []pullRequest{recordPR(92, "closed", "maintainer", "picked by hand"), recordPR(91, "closed", "maintainer", recordBody(91))}, carrier: 91}, + {name: "the bot's batch without a block", prs: []pullRequest{recordPR(92, "closed", bot, "edited away"), recordPR(91, "closed", bot, recordBody(91))}, bot: bot, refusal: "carries no ```backport-batch block"}, + {name: "a fork is no batch", prs: []pullRequest{fork, recordPR(91, "closed", "maintainer", recordBody(91))}, carrier: 91}, + {name: "a newer record by someone untrusted", prs: []pullRequest{recordPR(92, "closed", "someone", recordBody(92)), recordPR(91, "closed", "maintainer", recordBody(91))}, refusal: "#92"}, + {name: "a closed record newer than the open batch", prs: []pullRequest{recordPR(92, "closed", "maintainer", recordBody(92)), recordPR(91, "open", "maintainer", recordBody(91))}, open: 91, refusal: "newer than the open batch pull request #91"}, + {name: "a record on the second page", prs: append(func() (out []pullRequest) { + for n := 1000; n < 1000+recentPullsPage; n++ { + out = append(out, other(n)) + } + return out + }(), recordPR(91, "closed", "maintainer", recordBody(91))), carrier: 91, pages: 2}, + {name: "ten full pages without a record", prs: func() (out []pullRequest) { + for n := 1000; n < 1000+10*recentPullsPage+1; n++ { + out = append(out, other(n)) + } + return out + }(), refusal: "pull requests", pages: 10}, + {name: "the bot's record when no bot is trusted", prs: []pullRequest{recordPR(91, "closed", bot, recordBody(91))}, refusal: "#91"}, + {name: "a hand-written open batch beside a closed record", prs: []pullRequest{recordPR(92, "open", "maintainer", "picked by hand"), recordPR(91, "closed", "maintainer", recordBody(91))}, open: 92, carrier: 91}, + {name: "no batch ever", prs: []pullRequest{other(5)}}, + {name: "an unreadable record", prs: []pullRequest{recordPR(91, "closed", "maintainer", "text\n\n"+deferredFence+"\n{\n```\n")}, refusal: "#91 (closed)"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + l, _ := parseLine("release-1.1") + gh := &fakeGH{prs: tc.prs, bot: tc.bot} + var open *pullRequest + for i := range tc.prs { + if tc.prs[i].Number == tc.open { + open = &tc.prs[i] + } + } + src, err := findLineRecord(gh, l, open) + switch { + case tc.refusal != "": + if err == nil || !strings.Contains(err.Error(), tc.refusal) { + t.Errorf("findLineRecord = %+v, %v; want a refusal with %q", src, err, tc.refusal) + } + case err != nil: + t.Errorf("findLineRecord: %v", err) + case tc.carrier == 0 && (src.carrier != nil || src.rec != nil): + t.Errorf("found #%d, want no record", src.carrier.Number) + case tc.carrier != 0 && (src.carrier == nil || src.carrier.Number != tc.carrier || src.rec == nil || src.open != tc.isOpen || src.rec.Deferred[0].Original != tc.carrier): + t.Errorf("found %+v, want #%d open=%v", src, tc.carrier, tc.isOpen) + } + if tc.pages != 0 && gh.listed != tc.pages { + t.Errorf("read %d pages, want %d", gh.listed, tc.pages) + } + if tc.open != 0 && src.code != open { + t.Errorf("the batch to build on is %+v, want #%d", src.code, tc.open) + } + }) + } +} + +// A run that picks nothing while no batch is open writes nothing, since +// merged and closed batches are never written: it says its classification is +// not kept, and check still reads the merged batch's record. +func TestZeroPickRunLeavesNoRecord(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n", "app/seven.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mergeBatch(f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + edits, comments := len(gh.edited), len(gh.issueComments) + + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 0 || !strings.Contains(stderr, "no batch pull request is open on release-1.1; this run's classification is not retained") { + t.Errorf("run = %d, want 0 saying nothing is retained:\n%s", code, stderr) + } + if len(gh.created) != 1 || len(gh.edited) != edits || len(gh.issueComments) != comments || f.remoteHead(batchBranch1) != "" { + t.Errorf("created %q, edited %q, %d comments, branch at %q; want nothing written", gh.created, gh.edited, len(gh.issueComments), f.remoteHead(batchBranch1)) + } + f.git("switch", "-q", "main") + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 1 || !strings.Contains(stdout, "/pull/7 ") || strings.Contains(stdout, "/pull/5 ") || !strings.Contains(stderr, "deferred conflict on #90 (merged)") { + t.Errorf("check = %d, want 1 for #7 alone and #5 still a deferred conflict:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } +} + +// The open batch's author is asked for once although both the open batch and +// its record are read. +func TestRecordAsksForAPermissionOnce(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + gh.asked = nil + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 { + t.Fatalf("check = %d\n%s", code, stderr) + } + if len(gh.asked) != 1 { + t.Errorf("asked GitHub for %q, want the author once", gh.asked) + } +} + +// The record and the report name what the classification was made of: the +// heads it read, the policy, the tool, the job, the time and the flags. +func TestRecordNamesItsInputs(t *testing.T) { + f := newFixture(t) + f.lines() + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.onLine("release-1.1", "fix(app): five\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "5\n"}) + f.tool = strings.Repeat("c", 40) + "+dirty" + t.Setenv("GITHUB_SERVER_URL", "https://github.com") + t.Setenv("GITHUB_REPOSITORY", "cozystack/cozystack") + t.Setenv("GITHUB_RUN_ID", "123") + t.Setenv("GITHUB_RUN_ATTEMPT", "2") + gh := &fakeGH{} + stdout := mustRun(t, f, gh, "run", "--line", "release-1.0", "--no-workflow-files", "--push") + l, _ := parseLine("release-1.0") + const branch = "backport-batch/release-1.0-20261026" + rec, err := parseRecord(stdout, l) + if err != nil { + t.Fatal(err) + } + want := builtFrom{Main: f.git("rev-parse", "origin/main"), Line: f.git("rev-parse", "origin/release-1.0"), Newer: f.git("rev-parse", "origin/release-1.1"), + Policy: f.git("rev-parse", "origin/main"), Batch: f.remoteHead(branch), Tool: f.tool, Run: "https://github.com/cozystack/cozystack/actions/runs/123/attempts/2", + At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: true, Mode: modeNew}} + if rec.Version != recordVersion || *rec.Built != want || bodyReportOf(t, stdout, l).doc.Built != want || rec.Report != bodyReportOf(t, stdout, l).doc.ID { + t.Errorf("record = %+v, built %+v; want built %+v and the report it names", rec, rec.Built, want) + } + + // A policy from another ref, a run by hand that continues the batch. + f.git("push", "-q", "origin", "origin/main:refs/heads/policy-draft") + f.git("fetch", "-q", "origin") + t.Setenv("GITHUB_RUN_ID", "") + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + mc6 := f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.onLine("release-1.1", "fix(app): six\n\n(cherry picked from commit "+mc6+")", map[string]string{"app/six.txt": "6\n"}) + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--continue", "--policy", "origin/policy-draft", "--push") + if rec, err = parseRecord(stdout, l); err != nil { + t.Fatal(err) + } + if b := rec.Built; b.Policy != f.git("rev-parse", "origin/policy-draft") || b.Main != f.git("rev-parse", "origin/main") || b.Run != "" || b.Options != (runOptions{Mode: modeContinue}) || b.Batch != f.remoteHead(branch) { + t.Errorf("built %+v; want the draft's policy, no job and a continuation without the flag", b) + } +} + +// What a run assembled with --no-workflow-files stays saved through the +// operations that only change the record, which take no such flag: check +// keeps failing the cut on the wait they leave in place. +func TestMetadataOperationsKeepTheMode(t *testing.T) { + for _, v1 := range []bool{false, true} { + t.Run(fmt.Sprintf("an open record of version 1: %v", v1), func(t *testing.T) { + f := workflowWorld(t) + f.onLine("release-1.1", "fix(app): a line-only nine", map[string]string{"app/nine.txt": "line\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + f.pr(9, "fix(app): nine", "", map[string]string{"app/nine.txt": "9\n"}) + gh := &fakeGH{} + stdout := mustRun(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + reopen(f, gh) + if v1 { + waits := []pick{{v: &verdict{o: &original{number: 5}}, files: []string{".github/workflows/five.yaml"}}} + gh.prs[0].Body = strings.Replace(stdout, strings.Split(strings.Split(stdout, deferredFence+"\n")[1], "\n")[0], strings.Split(deferredBlock(notifyLine, nil, nil, waits), "\n")[1], 1) + } else if rec, err := parseRecord(stdout, notifyLine); err != nil || !rec.Built.Options.NoWorkflowFiles { + t.Fatalf("the scheduled run saved %+v, %v", rec, err) + } + for _, op := range []struct { + args []string + mode string + }{ + {[]string{"--drop", "9", "--reason", "needs #4"}, modeDrop}, + {[]string{"--undrop", "9"}, modeUndrop}, + {[]string{"--defer", "7", "--reason", "fails the e2e"}, modeDefer}, + } { + stdout = mustRun(t, f, gh, append(append([]string{"run", "--line", "release-1.1"}, op.args...), "--push")...) + rec, err := parseRecord(stdout, notifyLine) + if err != nil || rec.Built.Options != (runOptions{NoWorkflowFiles: true, Mode: op.mode}) { + t.Errorf("after %q the record is %+v, %v; want the flag kept", op.args, rec.Built, err) + } + reopen(f, gh) + } + var rec *lineRecord + var err error + reopen(f, gh) + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/") { + t.Errorf("check --cut = %d, want #5 still waiting:\n%s", code, stdout) + } + + // Deferring the last picks closes the batch, and its record + // still says how the batch was assembled. + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "6,8", "--reason", "fails the e2e", "--push") + rec, err = parseRecord(gh.find(90).Body, notifyLine) + if err != nil || gh.find(90).State != "closed" || rec.Closing == nil || rec.Built.Options != (runOptions{NoWorkflowFiles: true, Mode: modeClose}) { + t.Errorf("after the close the record is %+v, %v; want the flag kept", rec.Built, err) + } + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check --cut = %d, want #5 missing once no batch lists it:\n%s", code, stdout) + } + }) + } +} + +// A state is named only when it is one GitHub uses; anything else is quoted. +func TestPRStateNamesOnlyKnownStates(t *testing.T) { + at := "2026-10-26T12:00:00Z" + for _, tc := range []struct { + pr pullRequest + want string + }{ + {pullRequest{State: "open"}, "open"}, + {pullRequest{State: ""}, "open"}, + {pullRequest{State: "closed"}, "closed"}, + {pullRequest{State: "closed", MergedAt: &at}, "merged"}, + {pullRequest{State: "open\n\x1b[2Jforged"}, `in state "open\n\x1b[2Jforged"`}, + } { + if got := prState(&tc.pr); got != tc.want { + t.Errorf("prState(%q) = %q, want %q", tc.pr.State, got, tc.want) + } + } +} + +// v3 is v2 as version 3, with extra after the stale list. +func v3(extra string) string { + return strings.Replace(v2(extra), `"version":2`, `"version":3`, 1) +} + +// Version 3 adds what is parked and which units do only part of their +// original, each held to what a run writes, and version 2 has neither. +func TestRecordVersion3(t *testing.T) { + l, _ := parseLine("release-1.1") + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + sha, other := strings.Repeat("a", 40), strings.Repeat("c", 40) + parked := `,"parked":[{"original":10,"merge":"` + sha + `","kind":"partial","files":["a","b"]},{"original":11,"merge":"` + sha + `","kind":"dependency","needs":[3,4],"commits":["` + other + `"],"announced":true}],"partial":[9]` + waits := strings.Replace(v3(parked), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"files":[],"needs":[3,5]}`, 1) + rec, err := parseRecord(block(waits), l) + if err != nil || rec.Version != 3 || len(rec.Parked) != 2 || !rec.Parked[1].Announced || !slices.Equal(rec.Partial, []int{9}) || !slices.Equal(rec.deferred()[6].Needs, []int{3, 5}) { + t.Fatalf("parseRecord of a whole v3 record = %+v, %v", rec, err) + } + for name, json := range map[string]string{ + "version 1 with a wait": `{"line":"release-1.1","deferred":[{"original":6,"files":[],"needs":[3,5]}]}`, + "a wait with files": strings.Replace(waits, `"files":[],"needs":[3,5]`, `"files":["a"],"needs":[3,5]`, 1), + "a wait that is a conflict": strings.Replace(waits, `"files":[],"needs":[3,5]`, `"commit":"`+sha+`","files":[],"needs":[3,5]`, 1), + "a wait on itself": strings.Replace(waits, `"needs":[3,5]`, `"needs":[3,6]`, 1), + "an unsorted wait": strings.Replace(waits, `"needs":[3,5]`, `"needs":[5,3]`, 1), + "a wait on nothing": strings.Replace(waits, `"needs":[3,5]`, `"needs":[0]`, 1), + "a wait cycle": strings.Replace(strings.Replace(waits, `"needs":[3,5]`, `"needs":[12]`, 1), `"deferred":[`, `"deferred":[{"original":12,"files":[],"needs":[6]},`, 1), + "a wait on a parked one": strings.Replace(waits, `"needs":[3,5]`, `"needs":[10]`, 1), + "a wait on a dropped one": strings.Replace(waits, `"needs":[3,5]`, `"needs":[8]`, 1), + "a park on a deferred one": strings.Replace(waits, `"needs":[3,4]`, `"needs":[3,5]`, 1), + "a partial unit also deferred": strings.Replace(strings.Replace(waits, `"stale":[9]`, `"stale":[9,5]`, 1), `"partial":[9]`, `"partial":[9,5]`, 1), + "a changed pick also parked": v3(parked + `,"changed":[10]`), + "a changed pick also deferred": v3(parked + `,"changed":[5]`), + "a park on its own merge": strings.Replace(waits, `"commits":["`+other+`"]`, `"commits":["`+sha+`"]`, 1), + "a park cycle": strings.Replace(strings.Replace(waits, `"needs":[3,4]`, `"needs":[3,13]`, 1), `"parked":[`, `"parked":[{"original":13,"merge":"`+sha+`","kind":"dependency","needs":[11]},`, 1), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } + for name, json := range map[string]string{ + "version 2 with a park": v2(parked), + "version 4": strings.Replace(v3(""), `"version":3`, `"version":4`, 1), + "a park of a deferred original": v3(strings.Replace(parked, `"original":10`, `"original":5`, 1)), + "a park of a dropped original": v3(strings.Replace(parked, `"original":10`, `"original":8`, 1)), + "a partial park with no files": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":[]`, 1)), + "a partial park that needs": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["a"],"needs":[3]`, 1)), + "unsorted files": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["b","a"]`, 1)), + "a file that drives a terminal": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["a\u001b[2J"]`, 1)), + "a dependency park with no need": v3(strings.Replace(strings.Replace(parked, `"needs":[3,4],`, ``, 1), `"commits":["`+other+`"],`, ``, 1)), + "unsorted needs": v3(strings.Replace(parked, `"needs":[3,4]`, `"needs":[4,3]`, 1)), + "a park that needs itself": v3(strings.Replace(parked, `"needs":[3,4]`, `"needs":[3,11]`, 1)), + "a short commit": v3(strings.Replace(parked, `"commits":["`+other+`"]`, `"commits":["abc"]`, 1)), + "a short merge": v3(strings.Replace(parked, `"merge":"`+sha+`","kind":"partial"`, `"merge":"abc","kind":"partial"`, 1)), + "an unknown kind": v3(strings.Replace(parked, `"kind":"partial"`, `"kind":"later"`, 1)), + "a partial unit not on the batch": v3(strings.Replace(parked, `"partial":[9]`, `"partial":[10]`, 1)), + "a partial unit twice": v3(strings.Replace(parked, `"partial":[9]`, `"partial":[9,9]`, 1)), + "a changed pick twice": v3(parked + `,"changed":[4,4]`), + "a changed pick that is none": v3(parked + `,"changed":[0]`), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } +} diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go new file mode 100644 index 000000000..d1ae4a4b5 --- /dev/null +++ b/cmd/backport-batch/report.go @@ -0,0 +1,885 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "cmp" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "regexp" + "runtime/debug" + "slices" + "strconv" + "strings" + "time" +) + +// A report is one classification of a line: where every original merged into +// main since the line branched stands, and what it was made of. The batch +// pull request's body carries the latest one beside the record, and an issue +// comment the tool never edits keeps each one, so a later run that replaces +// the body loses none of them. Its ID hashes the classification, not the pull +// request it lands on, which a new batch does not have yet when the report is +// made. + +const ( + reportFence = "```backport-batch-report" + reportMarker = "\n" + + "
Backport batch report for " + s.doc.Line + ", " + date + " (" + s.doc.ID[:12] + ")\n\n" + + reportBlock(s.raw) + "\n
\n" + if len(c) > reportLimit { + return "", fmt.Errorf("the report comment is %d bytes, over the %d it may have", len(c), reportLimit) + } + return c, nil +} + +var reportMarkerRE = regexp.MustCompile(`^` + regexp.QuoteMeta(reportMarker) + `([0-9a-f]{64}) -->\n`) + +// findReports reads the report comments on batch pull request n, against l, +// by authors the batch trusts, as it trusts a record's author, by ID. One +// whose content does not match the ID its marker gives, or is not what a run +// writes, is returned apart, with why; a look-alike by anyone else is not +// read, whatever it says. +func findReports(gh github, n int, l line) (map[string]storedReport, map[string]string, error) { + comments, err := gh.comments(n) + if err != nil { + return nil, nil, err + } + found, bad := map[string]storedReport{}, map[string]string{} + for _, c := range comments { + if !strings.HasPrefix(c.Body, reportMarker) { + continue + } + why, err := trustedAuthor(gh, c.User.Login, c.User.Type) + if err != nil { + return nil, nil, err + } + if why != "" { + continue + } + m := reportMarkerRE.FindStringSubmatch(c.Body) + if m == nil { + bad[strconv.FormatInt(c.ID, 10)] = fmt.Sprintf("comment %d starts like a report but names none", c.ID) + continue + } + blocks, err := fencedLines(c.Body, reportFence) + if err == nil && len(blocks) != 1 { + err = fmt.Errorf("it holds %d reports", len(blocks)) + } + var doc reportDoc + if err == nil { + doc, err = parseReport(blocks[0], l) + } + if err == nil && doc.ID != m[1] { + err = fmt.Errorf("its marker names %.12s", m[1]) + } + if err != nil { + bad[m[1]] = fmt.Sprintf("comment %d: %v", c.ID, err) + continue + } + found[m[1]] = storedReport{doc: doc, raw: blocks[0]} + } + return found, bad, nil +} + +// toolVersion is the commit this binary was built from, with +dirty when its +// tree had changes, or "" when the build did not record it, as go run does +// not. +func toolVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + rev, dirty := "", false + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + rev = s.Value + case "vcs.modified": + dirty = s.Value == "true" + } + } + if checkSHA(rev) != nil { + return "" + } + if dirty { + rev += "+dirty" + } + return rev +} + +// runURL is the Actions run the command runs in, from the variables GitHub +// sets for it, or "" outside one. +func runURL(lookup func(string) (string, bool)) string { + get := func(key string) string { v, _ := lookup(key); return v } + url := get("GITHUB_SERVER_URL") + "/" + get("GITHUB_REPOSITORY") + "/actions/runs/" + get("GITHUB_RUN_ID") + if attempt := get("GITHUB_RUN_ATTEMPT"); attempt != "" && attempt != "1" { + url += "/attempts/" + attempt + } + if !runURLRE.MatchString(url) { + return "" + } + return url +} + +// report prints a report a batch pull request keeps, writing nothing, so it +// works on merged and closed batches too. It reads GitHub, and for a report +// that keeps drops also this clone's history at the heads the report was +// made from, which it never fetches. +func (a *app) report(args []string) int { + fs := a.flags("report") + number := fs.Int("pr", 0, "the batch pull request") + id := fs.String("id", "", "the report, by its ID (default: the one the body carries)") + asJSON := fs.Bool("json", false, "print the report's JSON") + if err := parse(fs, args); err != nil { + return 2 + } + if *number <= 0 { + return a.fail(fmt.Errorf("--pr %d is not a pull request number", *number)) + } + if *id != "" && !reportIDRE.MatchString(*id) { + return a.fail(fmt.Errorf("--id %q is not a report ID", *id)) + } + pr, err := a.gh.pull(*number) + if err != nil { + return a.fail(err) + } + l, err := parseLine(pr.Base.Ref) + if m := batchBranchRE.FindStringSubmatch(pr.Head.Ref); err != nil || m == nil || m[1] != l.branch() || pr.Head.Repo == nil || pr.Head.Repo.FullName != pr.Base.Repo.FullName { + return a.fail(fmt.Errorf("#%d is not a batch pull request of this repository", pr.Number)) + } + why, err := trusted(a.gh, pr) + if err != nil { + return a.fail(err) + } + if why != "" { + return a.fail(fmt.Errorf("batch pull request #%d was %s, so what it says is not the batch's", pr.Number, why)) + } + inBody, err := bodyReport(pr.Body, l) + if err != nil { + return a.fail(fmt.Errorf("batch pull request #%d: %w", pr.Number, err)) + } + if *id == "" { + if inBody == nil { + return a.fail(fmt.Errorf("batch pull request #%d carries no report in its body; name one with --id", pr.Number)) + } + *id = inBody.doc.ID + } + posted, bad, err := findReports(a.gh, pr.Number, l) + if err != nil { + return a.fail(err) + } + if why := bad[*id]; why != "" { + return a.fail(fmt.Errorf("batch pull request #%d: the report %.12s is not what its comment says: %s", pr.Number, *id, why)) + } + s, inComment := posted[*id] + switch { + case inComment: + case inBody != nil && inBody.doc.ID == *id: + s = *inBody + default: + return a.fail(fmt.Errorf("batch pull request #%d keeps no report %.12s", pr.Number, *id)) + } + if *asJSON { + fmt.Fprintln(a.stdout, s.raw) + return 0 + } + a.printReport(&pr, s, inBody, inComment) + return 0 +} + +// printReport prints a report. It was checked when it was read, and every +// field still goes through terminalSafe, so nothing it carries spans lines +// or drives a terminal. +func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, inComment bool) { + d, b := s.doc, s.doc.Built + safe := terminalSafe + fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode)) + if b.Options.NoWorkflowFiles { + fmt.Fprint(a.stdout, " with --no-workflow-files") + } + fmt.Fprintln(a.stdout) + fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none"))) + current := "none" + if inBody != nil { + current = safe(inBody.doc.ID) + } + switch { + case inBody != nil && inBody.doc.ID == d.ID: + fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number) + case pr.Head.SHA == b.Batch: + fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current) + default: + fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current) + } + if !inComment { + if prState(pr) == "open" { + fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it") + } else { + fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr)) + } + } + type line struct { + n int + text string + } + var lines []line + for _, n := range d.Primary.Picked { + lines = append(lines, line{n, "picked"}) + } + for _, n := range d.Primary.Missing { + lines = append(lines, line{n, "missing"}) + } + for _, e := range d.Primary.Deferred { + text := "deferred: " + switch { + case e.Workflows: + text += "waits for a maintainer, changes " + strings.Join(e.Files, ", ") + case len(e.Needs) > 0: + var needs []string + for _, n := range e.Needs { + needs = append(needs, "#"+strconv.Itoa(n)) + } + text += "waits on " + strings.Join(needs, ", ") + case e.Reason != "": + text += e.Reason + default: + text += fmt.Sprintf("conflict at %.12s in %s", e.Commit, strings.Join(e.Files, ", ")) + } + if e.More > 0 { + text += fmt.Sprintf(" and %d more", e.More) + } + lines = append(lines, line{e.Original, text}) + } + for _, e := range d.Primary.Parked { + text := "parked: " + switch { + case e.Kind == parkPartial: + text += "only part of it applies, it would leave " + strings.Join(e.Files, ", ") + " without effect" + default: + var needs []string + for _, n := range e.Needs { + needs = append(needs, "#"+strconv.Itoa(n)) + } + for _, c := range e.Commits { + needs = append(needs, c[:12]) + } + text += "builds on " + strings.Join(needs, ", ") + ", which never reaches the line" + } + lines = append(lines, line{e.Original, text}) + } + for _, l := range d.Primary.Skipped.lists() { + for _, n := range *l.list { + lines = append(lines, line{n, "skipped." + l.name}) + } + } + slices.SortFunc(lines, func(x, y line) int { return cmp.Compare(x.n, y.n) }) + for _, l := range lines { + fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text)) + } + overlay := func(name string, list []int) { + if len(list) > 0 { + fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]")) + } + } + overlay("stale, still on the batch", d.Overlays.Stale) + overlay("only part of it applies, still on the batch", d.Overlays.Partial) + for _, e := range d.Overlays.Dropped { + fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason)) + } + overlay("changed in the pick", d.Overlays.Changed) + for _, n := range slices.Sorted(maps.Keys(d.Diag)) { + for _, ln := range d.Diag[n] { + fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln)) + } + } + if len(d.Other) > 0 { + other := make([]string, len(d.Other)) + for i, sha := range d.Other { + other[i] = safe(sha) + } + fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", ")) + } +} + +func or(s, otherwise string) string { + if s == "" { + return otherwise + } + return s +} + +// postReport writes a report comment on pull request n, after GitHub says +// it is still open: nothing is written to a merged or closed one. +func (a *app) postReport(n int, s storedReport) error { + if err := a.stillOpen(n); err != nil { + return err + } + c, err := reportComment(s) + if err != nil { + return err + } + name, err := writeTemp("backport-batch-report-*.md", c) + if err != nil { + return err + } + defer os.Remove(name) + return a.gh.createComment(n, name) +} + +// stillOpen reads pull request n again and refuses to go on unless it is +// open. +func (a *app) stillOpen(n int) error { + pr, err := a.gh.pull(n) + if err != nil { + return err + } + if st := prState(&pr); st != "open" { + return fmt.Errorf("batch pull request #%d is %s now; nothing is written to it", n, st) + } + return nil +} + +func writeTemp(pattern, content string) (string, error) { + f, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + if _, err := f.WriteString(content); err != nil { + f.Close() + os.Remove(f.Name()) + return "", err + } + if err := f.Close(); err != nil { + os.Remove(f.Name()) + return "", err + } + return f.Name(), nil +} diff --git a/cmd/backport-batch/report_test.go b/cmd/backport-batch/report_test.go new file mode 100644 index 000000000..a496f68e0 --- /dev/null +++ b/cmd/backport-batch/report_test.go @@ -0,0 +1,778 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + "unicode" +) + +// reportsPosted lists the report IDs the comments carry, in the order they +// were posted, whoever wrote them. +func reportsPosted(gh *fakeGH) []string { + var ids []string + for _, c := range gh.issueComments { + if m := reportMarkerRE.FindStringSubmatch(c.Body); m != nil { + ids = append(ids, m[1]) + } + } + return ids +} + +// bodyReportOf is the report a body on release-1.1, or on the line given, +// carries, failing the test without one. +func bodyReportOf(t *testing.T, body string, on ...line) storedReport { + t.Helper() + l := notifyLine + if len(on) > 0 { + l = on[0] + } + s, err := bodyReport(body, l) + if err != nil || s == nil { + t.Fatalf("the body carries no readable report: %v\n%s", err, body) + } + return *s +} + +// refusePushes makes the remote refuse every push, as a lost connection or +// a protected branch would, until it is called with false. +func (f *fixture) refusePushes(on bool) { + f.t.Helper() + hook := filepath.Join(f.remote, "hooks", "pre-receive") + if !on { + if err := os.Remove(hook); err != nil { + f.t.Fatal(err) + } + return + } + if err := os.WriteFile(hook, []byte("#!/bin/sh\necho refused >&2\nexit 1\n"), 0o755); err != nil { + f.t.Fatal(err) + } +} + +func mustRun(t *testing.T, f *fixture, gh *fakeGH, args ...string) string { + t.Helper() + code, stdout, stderr := f.exec(gh, args...) + if code != 0 { + t.Fatalf("%q = %d\n%s", args, code, stderr) + } + f.git("switch", "-q", "main") + return stdout +} + +// A report comment keeps each classification: one the next run replaced in +// the body is still read back by its ID, also once the batch merged, which +// nothing writes to. +func TestReportSurvivesContinuation(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n"}) + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + r1 := bodyReportOf(t, gh.prs[0].Body).doc.ID + f.onLine("release-1.1", "fix(app): five, by hand\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "5\n"}) + f.now = f.now.AddDate(0, 0, 1) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + r2 := bodyReportOf(t, gh.find(90).Body).doc.ID + if r1 == r2 || !slices.Equal(reportsPosted(gh), []string{r1, r2}) { + t.Fatalf("reports %q; want R1 %.12s, then R2 %.12s", reportsPosted(gh), r1, r2) + } + read := func(id, want string) { + t.Helper() + code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", id) + if code != 0 || !strings.Contains(stdout, want) { + t.Errorf("report --id %.12s = %d, want %q:\n%s%s", id, code, want, stdout, stderr) + } + } + for _, merged := range []bool{false, true} { + if merged { + mergeBatch(f, gh) + } + read(r1, "#5 deferred: conflict at ") + read(r2, "#5 skipped.on-line") + read(r2, "#6 picked") + } +} + +// Two runs that find the same post one report, and the body carries the one +// kept, whatever time and job the second ran in. A look-alike someone else +// wrote is no report, and a kept report edited since stops the reader. +func TestReportDedupAndTrust(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + t.Setenv("GITHUB_SERVER_URL", "https://github.com") + t.Setenv("GITHUB_REPOSITORY", "cozystack/cozystack") + t.Setenv("GITHUB_RUN_ID", "100") + t.Setenv("GITHUB_RUN_ATTEMPT", "1") + + // The report the next run makes, read off a run that publishes nothing, + // planted by someone the batch does not trust. + preview := bodyReportOf(t, mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue")) + planted, err := reportComment(preview) + if err != nil { + t.Fatal(err) + } + look := issueComment{ID: 7, Body: planted} + look.User.Login, look.User.Type = "someone", "User" + gh.issueComments = append(gh.issueComments, look) + + f.now = f.now.AddDate(0, 0, 1) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + kept := bodyReportOf(t, gh.find(90).Body) + if kept.doc.ID != preview.doc.ID { + t.Fatalf("the run made %.12s, the preview %.12s", kept.doc.ID, preview.doc.ID) + } + f.now = f.now.AddDate(0, 0, 1) + t.Setenv("GITHUB_RUN_ID", "101") + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + posted := reportsPosted(gh) + if n := len(slices.DeleteFunc(slices.Clone(posted), func(id string) bool { return id != kept.doc.ID })); n != 2 || len(posted) != 3 { + t.Errorf("reports %q; want the first, the look-alike and the run's own once", posted) + } + body := gh.find(90).Body + again := bodyReportOf(t, body) + rec, err := parseRecord(body, notifyLine) + if err != nil || again.raw != kept.raw || rec.Report != again.doc.ID || !strings.Contains(again.raw, `"run":"https://github.com/cozystack/cozystack/actions/runs/100"`) { + t.Errorf("the second run's body carries %s with record report %v (%v); want the kept bytes of run 100", again.raw, rec, err) + } + + // The reader takes the run's own comment, not the look-alike. + if code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", kept.doc.ID); code != 0 || !strings.Contains(stdout, "#6 picked") { + t.Errorf("report = %d:\n%s%s", code, stdout, stderr) + } + for i := range gh.issueComments { + c := &gh.issueComments[i] + if c.User.Login == "maintainer" && strings.Contains(c.Body, kept.doc.ID) { + c.Body = strings.Replace(c.Body, `"picked":[5,6]`, `"picked":[5]`, 1) + } + } + if code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", kept.doc.ID); code != 2 || !strings.Contains(stderr, "does not match") { + t.Errorf("report of an edited comment = %d, want 2:\n%s%s", code, stdout, stderr) + } +} + +// The report holds no pull request number, so a new batch's report is whole +// before GitHub numbers the pull request, and the reader finds it on +// whatever number it got. +func TestFirstReportIDIsKnownBeforeTheNumber(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{nextPR: 4242} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + inBody := bodyReportOf(t, gh.bodies[0]) + // Every list is one, empty or not, so equal reports are equal bytes: + // main here has no commit that merged no pull request. + if !strings.Contains(inBody.raw, `"other":[]`) { + t.Errorf("the report writes its empty list of other commits as something else:\n%s", inBody.raw) + } + if !slices.Equal(gh.commented, []int{4242}) || !slices.Equal(reportsPosted(gh), []string{inBody.doc.ID}) { + t.Fatalf("commented on %v with %q, want #4242 with %.12s", gh.commented, reportsPosted(gh), inBody.doc.ID) + } + found, bad, err := findReports(gh, 4242, notifyLine) + if err != nil || len(bad) != 0 || found[inBody.doc.ID].raw != inBody.raw { + t.Errorf("findReports = %v, %v, %v; want the body's report", found, bad, err) + } + if code, stdout, stderr := f.exec(gh, "report", "--pr", "4242"); code != 0 || !strings.Contains(stdout, "report "+inBody.doc.ID) || !strings.Contains(stdout, "#4242's body carries this report") { + t.Errorf("report --pr 4242 = %d:\n%s%s", code, stdout, stderr) + } +} + +// Every original main merged since the line branched is in exactly one +// category of the report, a commit that merged no pull request is counted +// apart, and the report says so whatever the run had to do. +func TestReportPartition(t *testing.T) { + f := newFixture(t) + f.lines() + gh := &fakeGH{labels: map[string][]int{labelSkip: {9}}} + // onNewer puts an original on release-1.1, as the newer line's batch + // would, so rule 6 lets it onto release-1.0. + onNewer := func(mc string, files map[string]string) { + f.onLine("release-1.1", "fix: by the newer batch\n\n(cherry picked from commit "+mc+")", files) + } + pr := func(n int, title string, files map[string]string, newer bool) { + mc := f.pr(n, title, "", files) + if newer { + onNewer(mc, files) + } + } + t.Setenv("GIT_COMMITTER_DATE", "2026-01-01T00:00:00Z") + f.pr(4, "fix(app): four, before the start", "", map[string]string{"app/four.txt": "4\n"}) + os.Unsetenv("GIT_COMMITTER_DATE") + f.setPolicy("release-1.0: 2026-06-01\n") + f.onLine("release-1.0", "fix(app): a line-only eleven", map[string]string{"app/eleven.txt": "line\n"}) + pr(5, "fix(app): five", map[string]string{"app/five.txt": "5\n"}, true) + pr(6, "fix(app): six, not on the newer line", map[string]string{"app/six.txt": "6\n"}, false) + pr(7, "fix(app)!: seven", map[string]string{"app/seven.txt": "7\n"}, true) + pr(8, "fix(platform): eight", map[string]string{migrationsDir + "42": "#!/bin/sh\n"}, true) + pr(9, "fix(app): nine", map[string]string{"app/nine.txt": "9\n"}, true) + mc10 := f.pr(10, "fix(app): ten", "", map[string]string{"app/ten.txt": "10\n"}) + onNewer(mc10, map[string]string{"app/ten.txt": "10\n"}) + f.onLine("release-1.0", "fix(app): ten\n\n(cherry picked from commit "+mc10+")", map[string]string{"app/ten.txt": "10\n"}) + pr(11, "fix(app): eleven", map[string]string{"app/eleven.txt": "11\n"}, true) + pr(12, "fix(ci): twelve", map[string]string{".github/workflows/twelve.yaml": "on: push\n"}, true) + pr(13, "fix(app): thirteen", map[string]string{"app/thirteen.txt": "13\n"}, true) + f.git("switch", "-q", "main") + f.write("app/direct.txt", "pushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: a direct push") + direct := f.git("rev-parse", "HEAD") + f.git("push", "-q", "origin", "main") + + stdout := mustRun(t, f, gh, "run", "--line", "release-1.0", "--no-workflow-files", "--push") + const branch = "backport-batch/release-1.0-20261026" + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + pr(14, "fix(app): fourteen, after the batch", map[string]string{"app/fourteen.txt": "14\n"}, true) + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--defer", "13", "--reason", "fails the e2e", "--push") + // An active drop, of #5, which the batch carries, and one of a feat, + // which rule 2 keeps out first. + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--drop", "5,1", "--reason", "not for 1.0", "--push") + + l, _ := parseLine("release-1.0") + s := bodyReportOf(t, stdout, l) + got, err := s.doc.categories() + if err != nil { + t.Fatal(err) + } + want := map[int]string{1: "skipped.type", 4: "skipped.before-start", 5: "skipped.dropped", 6: "skipped.newer", 7: "skipped.breaking", + 8: "skipped.touches", 9: "skipped.skip-label", 10: "skipped.on-line", 11: "deferred", 12: "deferred", 13: "deferred", + 14: "missing", 9000: "skipped.type"} + for n, category := range want { + if got[n] != category { + t.Errorf("#%d is %q, want %q", n, got[n], category) + } + delete(got, n) + } + if len(got) != 0 { + t.Errorf("the report also names %v", got) + } + if gh.find(90).State != "closed" || !strings.Contains(stdout, `"mode":"close"`) { + t.Errorf("dropping the batch's last pick left #90 %s; want it closed", gh.find(90).State) + } + if !slices.Equal(droppedNumbers(s.doc.Overlays.Dropped), []int{1, 5}) || len(s.doc.Primary.Picked) != 0 { + t.Errorf("the report drops %v and picks %v; want #1 and #5 dropped, nothing picked", s.doc.Overlays.Dropped, s.doc.Primary.Picked) + } + if !slices.Contains(s.doc.Other, direct) { + t.Errorf("the report does not count the direct push %.12s: %v", direct, s.doc.Other) + } + kinds := map[int]string{} + for _, e := range s.doc.Primary.Deferred { + kinds[e.Original] = fmt.Sprintf("commit=%v reason=%q workflows=%v", e.Commit != "", e.Reason, e.Workflows) + } + if kinds[11] != `commit=true reason="" workflows=false` || kinds[12] != `commit=false reason="" workflows=true` || kinds[13] != `commit=false reason="fails the e2e" workflows=false` { + t.Errorf("the deferred entries are %v", kinds) + } +} + +// A report too big for a comment is refused before anything is written: no +// push, no comment, no edit, and no replay of the earlier report either. +func TestReportTooLargeIsRefusedBeforePush(t *testing.T) { + f, gh, head := bigWaits(t, 7) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push") + if code != 2 || !strings.Contains(stderr, "the report comment is") { + t.Errorf("run --push = %d, want a refusal of the report:\n%s", code, stderr) + } + if f.remoteHead(batchBranch1) != head || len(gh.issueComments) != 0 || len(gh.edited) != 0 { + t.Errorf("the refused run moved the batch to %.12s, wrote %d comments and edited %q", f.remoteHead(batchBranch1), len(gh.issueComments), gh.edited) + } +} + +// bigWaits opens a batch whose report comment failed, so its body keeps a +// report no comment does, and then merges n originals that each wait on 50 +// workflow files with long names, which both the record and the report list +// in full. +func bigWaits(t *testing.T, n int) (*fixture, *fakeGH, string) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{fail: map[string]error{"createComment": errString("HTTP 502")}} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 2 || len(gh.created) != 1 { + t.Fatalf("the first run = %d, want its comment to fail after the pull request opened\n%s", code, stderr) + } + f.git("switch", "-q", "main") + for i := range n { + files := map[string]string{} + for j := range filesKept { + files[fmt.Sprintf(".github/workflows/%s-%02d-%02d.yaml", strings.Repeat("w", 170), i, j)] = "on: push\n" + } + f.pr(10+i, fmt.Sprintf("fix(ci): wait %d", i), "", files) + } + f.now = f.now.AddDate(0, 0, 1) + return f, gh, f.remoteHead(batchBranch1) +} + +// A failure between any two writes leaves what was written readable, and the +// next run, whatever changed meanwhile, replays a report the body kept but no +// comment did, byte for byte, and posts nothing twice. A batch closed in +// between is never written again. +func TestReportReplayAfterEachFailure(t *testing.T) { + setup := func(t *testing.T) (*fixture, *fakeGH) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f, &fakeGH{} + } + // change moves a label, main, the clock and the run's flags between + // the failure and the retry, so the retry's report is another one. + change := func(f *fixture, gh *fakeGH) []string { + gh.labels = map[string][]int{labelSkip: {6}} + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.now = f.now.AddDate(0, 0, 1) + return []string{"run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push"} + } + once := func(t *testing.T, gh *fakeGH, ids ...string) { + t.Helper() + posted := reportsPosted(gh) + for _, id := range ids { + if n := len(slices.DeleteFunc(slices.Clone(posted), func(p string) bool { return p != id })); n != 1 { + t.Errorf("report %.12s is posted %d times: %q", id, n, posted) + } + } + } + + t.Run("a new batch whose pull request was not opened", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if len(gh.prs) != 0 || len(gh.issueComments) != 0 || f.remoteHead(batchBranch1) == "" { + t.Errorf("after the failure: %d pull requests, %d comments, branch %q; want the branch alone", len(gh.prs), len(gh.issueComments), f.remoteHead(batchBranch1)) + } + mustRun(t, f, gh, change(f, gh)...) + if len(gh.prs) != 1 || len(reportsPosted(gh)) != 1 || bodyReportOf(t, gh.prs[0].Body).doc.ID != reportsPosted(gh)[0] { + t.Errorf("the retry opened %d pull requests with reports %q", len(gh.prs), reportsPosted(gh)) + } + }) + t.Run("a new batch whose report comment failed", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 || !strings.Contains(stderr, "the body keeps it") { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + first := bodyReportOf(t, gh.find(90).Body) + mustRun(t, f, gh, change(f, gh)...) + second := bodyReportOf(t, gh.find(90).Body) + want, err := reportComment(first) + if err != nil || len(gh.issueComments) != 2 || gh.issueComments[0].Body != want || reportsPosted(gh)[1] != second.doc.ID || first.doc.ID == second.doc.ID { + t.Errorf("comments %q; want the first report replayed as it was, then the retry's", reportsPosted(gh)) + } + }) + t.Run("a continuation whose push failed", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + first := reportsPosted(gh)[0] + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + f.refusePushes(true) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + f.refusePushes(false) + attempted := reportsPosted(gh)[1] + if bodyReportOf(t, gh.find(90).Body).doc.ID != first { + t.Error("the body moved on although the push failed") + } + mustRun(t, f, gh, change(f, gh)...) + once(t, gh, first, attempted) + if len(reportsPosted(gh)) != 3 { + t.Errorf("reports %q; want the first, the attempted and the retry's", reportsPosted(gh)) + } + }) + t.Run("a continuation whose body edit failed", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + first := reportsPosted(gh)[0] + before := f.remoteHead(batchBranch1) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.fail = map[string]error{"editPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + attempted := reportsPosted(gh)[1] + if f.remoteHead(batchBranch1) == before || bodyReportOf(t, gh.find(90).Body).doc.ID != first { + t.Error("the push or the body is not where the failure left them") + } + mustRun(t, f, gh, change(f, gh)...) + once(t, gh, first, attempted) + if last := reportsPosted(gh)[len(reportsPosted(gh))-1]; bodyReportOf(t, gh.find(90).Body).doc.ID != last || len(reportsPosted(gh)) != 3 { + t.Errorf("reports %q; want the body to carry the retry's, posted last", reportsPosted(gh)) + } + }) + t.Run("a batch closed while the run writes to it", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.after = map[string]func(){"createComment": func() { gh.find(90).State = "closed" }} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || len(gh.edited) != 0 { + t.Errorf("run = %d, edited %q; want 2 and no edit of the closed pull request:\n%s", code, gh.edited, stderr) + } + }) + t.Run("a batch closed before its report was posted", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + kept := bodyReportOf(t, gh.find(90).Body) + gh.find(90).State = "closed" + gh.nextPR = 91 + mustRun(t, f, gh, change(f, gh)...) + if slices.Contains(gh.commented, 90) || slices.Contains(reportsPosted(gh), kept.doc.ID) { + t.Errorf("commented on %v, posting %q; want nothing on the closed #90", gh.commented, reportsPosted(gh)) + } + code, stdout, stderr := f.exec(gh, "report", "--pr", "90") + if code != 0 || !strings.Contains(stdout, "report "+kept.doc.ID) || !strings.Contains(stdout, "kept in the body only") { + t.Errorf("report --pr 90 = %d:\n%s%s", code, stdout, stderr) + } + }) + // Every comment is posted only after GitHub says the batch is still + // open, the replay's included, so a close between the two stops the + // second. + t.Run("a batch closed between the replay and the new report", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + kept := bodyReportOf(t, gh.find(90).Body) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.after = map[string]func(){"createComment": func() { gh.find(90).State = "closed" }} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || !slices.Equal(reportsPosted(gh), []string{kept.doc.ID}) || len(gh.edited) != 0 { + t.Errorf("run = %d, posted %q, edited %q; want 2 after the replay alone:\n%s", code, reportsPosted(gh), gh.edited, stderr) + } + }) +} + +// The reader says how a report stands against the pull request as GitHub +// shows it now, and claims publication only when the head or the body shows +// it: a failed push, a push whose answer was lost, a body that was not +// updated and a rebuild since all read as they are. +func TestReportPublicationStatus(t *testing.T) { + setup := func(t *testing.T) (*fixture, *fakeGH, string) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + return f, gh, reportsPosted(gh)[0] + } + status := func(t *testing.T, f *fixture, gh *fakeGH, id, want string) { + t.Helper() + code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", id) + if code != 0 || !strings.Contains(stdout, want) { + t.Errorf("report --id %.12s = %d, want %q:\n%s%s", id, code, want, stdout, stderr) + } + } + t.Run("a push the remote refused, and one it took whose answer was lost", func(t *testing.T) { + f, gh, first := setup(t) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.refusePushes(true) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.refusePushes(false) + attempted := reportsPosted(gh)[1] + status(t, f, gh, attempted, "publication unconfirmed: #90's head is ") + status(t, f, gh, attempted, "its body carries report "+first[:12]) + // The same head reaching the remote is what a push GitHub took looks + // like when its answer never came back. + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + status(t, f, gh, attempted, "the batch this report describes, but its body carries report "+first[:12]) + }) + t.Run("a push followed by a failed body edit", func(t *testing.T) { + f, gh, first := setup(t) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh.fail = map[string]error{"editPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + status(t, f, gh, reportsPosted(gh)[1], "the batch this report describes, but its body carries report "+first[:12]) + status(t, f, gh, first, "#90's body carries this report") + }) + t.Run("a rebuild since", func(t *testing.T) { + f, gh, first := setup(t) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "x", "--push") + second := reportsPosted(gh)[1] + status(t, f, gh, first, "publication unconfirmed: #90's head is ") + status(t, f, gh, second, "#90's body carries this report") + }) +} + +// forged is a report on release-1.1 with what change sets, under an ID that +// matches it: only checking what it says stands in its way. +func forged(change func(d *reportDoc)) storedReport { + sha := strings.Repeat("a", 40) + d := reportDoc{Format: reportFormat, Line: "release-1.1", Diag: map[string][]string{}, Other: []string{sha}, + Built: builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeNew}}} + d.Primary.Picked = []int{5} + change(&d) + d.ID = reportID(d) + return storedReport{doc: d, raw: reportJSON(d)} +} + +// reportRun runs report with gh faked; it reads nothing but GitHub. +func reportRun(gh *fakeGH, args ...string) (int, string, string) { + var stdout, stderr strings.Builder + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 0, 0, 0, time.UTC) }} + code := a.main(append([]string{"report"}, args...)) + return code, stdout.String(), stderr.String() +} + +// A report is read only when everything it says is what a run writes, and +// it is for the line of the pull request it sits on: an ID that matches its +// content proves only that nobody changed it since it was hashed. +func TestReportRefusesWhatItCannotVouchFor(t *testing.T) { + for name, change := range map[string]func(d *reportDoc){ + "a line that is no release line": func(d *reportDoc) { d.Line = "release-1.1\n\x1b[2Jpicked" }, + "another line": func(d *reportDoc) { d.Line = "release-1.0" }, + "a main that is no commit": func(d *reportDoc) { d.Built.Main = "main\n#5 picked" }, + "a time that is none": func(d *reportDoc) { d.Built.At = "2026-10-26T" }, + "an unknown operation": func(d *reportDoc) { d.Built.Options.Mode = "new\x1b[2J" }, + "a run that is no Actions run": func(d *reportDoc) { d.Built.Run = "javascript:alert(1)" }, + "a tool that is no commit": func(d *reportDoc) { d.Built.Tool = "v1\n" }, + "a commit on main that is none": func(d *reportDoc) { d.Other = []string{"HEAD\x1b]0;x\x07"} }, + "a deferred commit that is none": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 6, Commit: "-x", Files: []string{"a"}}} + }, + "an original named twice": func(d *reportDoc) { d.Primary.Missing = []int{5} }, + "an original that is none": func(d *reportDoc) { d.Primary.Skipped.Type = []int{-1} }, + "a diagnostic of two lines": func(d *reportDoc) { d.Diag = map[string][]string{"5": {"incomplete\n\x1b[2Jforged"}} }, + "a diagnostic of no original": func(d *reportDoc) { d.Diag = map[string][]string{"five": {"incomplete"}} }, + // Fields that speak of one another have to agree. + "a drop category with no drop": func(d *reportDoc) { d.Primary.Skipped.Dropped = []int{6} }, + "a stale original it has missing": func(d *reportDoc) { + d.Primary.Missing, d.Overlays.Stale = []int{7}, []int{7} + }, + "a stale original it puts nowhere": func(d *reportDoc) { d.Overlays.Stale = []int{8} }, + "a changed original it defers": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 9, Files: []string{}, Reason: "fails the e2e"}} + d.Overlays.Changed = []int{9} + }, + "a skip label on a picked original": func(d *reportDoc) { d.Overrides.Skip = []int{5} }, + "a skip-label category with no label": func(d *reportDoc) { d.Primary.Skipped.SkipLabel = []int{10} }, + "an include label on a rule 2 skip": func(d *reportDoc) { + d.Primary.Skipped.Type, d.Overrides.Include = []int{11}, []int{11} + }, + "both labels on one original": func(d *reportDoc) { + d.Primary.Skipped.SkipLabel, d.Overrides.Include, d.Overrides.Skip = []int{12}, []int{12}, []int{12} + }, + "a label on no original": func(d *reportDoc) { d.Overrides.Include = []int{13} }, + "a diagnostic of an original it puts nowhere": func(d *reportDoc) { d.Diag = map[string][]string{"14": {"incomplete"}} }, + "a workflow wait without the flag": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 15, Files: []string{".github/workflows/ci.yaml"}, Workflows: true}} + }, + "a close that still picks": func(d *reportDoc) { d.Built.Options.Mode = modeClose }, + "a park with no merge": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Kind: parkPartial, Files: []string{"a"}}} + }, + "a parked original it picks": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 5, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"a"}}} + }, + "a park of an unknown kind": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: "later", Files: []string{"a"}}} + }, + "a partial park that needs an original": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"a"}, Needs: []int{5}}} + }, + "a partial park of unsorted files": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"b", "a"}}} + }, + "a partial park of a file that drives a terminal": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"a\x1b[2J"}}} + }, + "a park announced in the report": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"a"}, Announced: true}} + }, + "a park for an original it picks": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkDependency, Needs: []int{5}}} + }, + "a park that needs itself": func(d *reportDoc) { + d.Primary.Skipped.Type = []int{17} + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkDependency, Needs: []int{16, 17}}} + }, + "a park for a commit that merged a pull request": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkDependency, Commits: []string{strings.Repeat("c", 40)}}} + }, + "a partial unit not on the batch": func(d *reportDoc) { d.Overlays.Partial = []int{5} }, + "a wait on an original it picks": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 6, Files: []string{}, Needs: []int{5}}} + }, + "a wait on an original it puts nowhere": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 6, Files: []string{}, Needs: []int{19}}} + }, + "a park on its own merge as a commit": func(d *reportDoc) { + d.Other = append(d.Other, strings.Repeat("b", 40)) + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkDependency, Commits: []string{strings.Repeat("b", 40)}}} + }, + "a park cycle": func(d *reportDoc) { + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkDependency, Needs: []int{18}}, {Original: 18, Merge: strings.Repeat("b", 40), Kind: parkDependency, Needs: []int{16}}} + }, + "a wait cycle": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 6, Files: []string{}, Needs: []int{7}}, {Original: 7, Files: []string{}, Needs: []int{6}}} + }, + "format 1 with a wait": func(d *reportDoc) { + d.Format = 1 + d.Primary.Missing = []int{19} + d.Primary.Deferred = []deferredEntry{{Original: 6, Files: []string{}, Needs: []int{19}}} + }, + "format 1 with a park": func(d *reportDoc) { + d.Format = 1 + d.Primary.Parked = []parkedEntry{{Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"a"}}} + }, + } { + s := forged(change) + c, _ := reportComment(storedReport{doc: forged(func(*reportDoc) {}).doc, raw: s.raw}) + comment := issueComment{ID: 1, Body: strings.Replace(c, forged(func(*reportDoc) {}).doc.ID, s.doc.ID, 1)} + comment.User.Login, comment.User.Type = "maintainer", "User" + for where, gh := range map[string]*fakeGH{ + "in the body": {prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(s.raw)+"\n")}}, + "in a comment": {prs: []pullRequest{notifyPR("20261026", "text")}, issueComments: []issueComment{comment}}, + } { + if code, stdout, stderr := reportRun(gh, "--pr", "90", "--id", s.doc.ID); code != 2 || stdout != "" { + t.Errorf("%s, %s: report = %d, want 2 and nothing printed:\n%s%s", name, where, code, stdout, stderr) + } + } + } + if code, stdout, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(forged(func(*reportDoc) {}).raw)+"\n")}}, "--pr", "90"); code != 0 || !strings.Contains(stdout, "#5 picked") { + t.Errorf("report of a sound one = %d:\n%s%s", code, stdout, stderr) + } + // What a run wrote before anything was parked reads as it was written. + format1 := forged(func(d *reportDoc) { d.Format, d.Primary.Parked = 1, []parkedEntry{} }) + if code, stdout, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(format1.raw)+"\n")}}, "--pr", "90"); code != 0 || !strings.Contains(stdout, "#5 picked") { + t.Errorf("report of a sound format 1 one = %d:\n%s%s", code, stdout, stderr) + } + parked := forged(func(d *reportDoc) { + d.Primary.Skipped.Type = []int{17} + d.Other = append(d.Other, strings.Repeat("c", 40)) + d.Primary.Parked = []parkedEntry{ + {Original: 16, Merge: strings.Repeat("b", 40), Kind: parkPartial, Files: []string{"chart/templates/policy.yaml"}}, + {Original: 18, Merge: strings.Repeat("b", 40), Kind: parkDependency, Needs: []int{16, 17}, Commits: []string{strings.Repeat("c", 40)}}, + } + }) + if code, stdout, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(parked.raw)+"\n")}}, "--pr", "90"); code != 0 || !strings.Contains(stdout, "#16 parked: only part of it applies") || !strings.Contains(stdout, "#18 parked: builds on #16, #17, cccccccccccc") { + t.Errorf("report of sound parks = %d:\n%s%s", code, stdout, stderr) + } +} + +// What report prints stays one line per field and carries nothing a +// terminal acts on, whatever reached it. +func TestReportPrintsNothingThatDrivesATerminal(t *testing.T) { + const evil = "\n\x1b[2Jfake: #9 picked\u202e" + s := forged(func(d *reportDoc) { + d.Line, d.Built.At, d.Built.Options.Mode = "release-1.1"+evil, "2026-10-26"+evil, "new"+evil + d.Built.Main, d.Built.Tool, d.Built.Run, d.Built.Newer = "main"+evil, "tool"+evil, "run"+evil, "newer"+evil + d.Other = []string{"other" + evil} + d.Primary.Deferred = []deferredEntry{{Original: 6, Reason: "why" + evil}} + }) + var stdout strings.Builder + a := &app{stdout: &stdout} + pr := notifyPR("20261026", "") + a.printReport(&pr, s, nil, true) + for _, ln := range strings.Split(stdout.String(), "\n") { + if strings.HasPrefix(ln, "fake") || strings.ContainsFunc(ln, func(r rune) bool { return r != '\t' && unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) { + t.Errorf("report printed %q", ln) + } + } +} + +// The comment is made of what was checked: a line, a time and an ID that +// carry no markup into the summary GitHub renders. +func TestReportCommentIsBuiltFromCheckedFields(t *testing.T) { + for name, change := range map[string]func(d *reportDoc){ + "markup in the line": func(d *reportDoc) { d.Line = "release-1.1" }, + "markup in the time": func(d *reportDoc) { d.Built.At = "2026-10-26T06:00:00Z" }, + "a mention": func(d *reportDoc) { d.Line = "release-1.1 @cozystack/maintainers" }, + } { + if c, err := reportComment(forged(change)); err == nil { + t.Errorf("%s: reportComment made\n%s", name, c) + } + } + if _, err := reportComment(forged(func(*reportDoc) {})); err != nil { + t.Errorf("reportComment of a sound report: %v", err) + } +} + +// Anyone can comment on a public pull request: the reader takes a report +// from a comment only when the batch trusts its author, whatever ID it +// carries. +func TestReportReaderTakesOnlyTrustedComments(t *testing.T) { + s := forged(func(*reportDoc) {}) + c, err := reportComment(s) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + login, kind string + read bool + }{ + {"someone", "User", false}, + {"renovate[bot]", "Bot", false}, + {"github-actions[bot]", "Bot", false}, + {"maintainer", "User", true}, + {defaultBatchBot, "Bot", true}, + } { + comment := issueComment{ID: 1, Body: c} + comment.User.Login, comment.User.Type = tc.login, tc.kind + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", "text")}, issueComments: []issueComment{comment}, bot: defaultBatchBot} + code, stdout, stderr := reportRun(gh, "--pr", "90", "--id", s.doc.ID) + if read := code == 0 && strings.Contains(stdout, "#5 picked"); read != tc.read || !tc.read && (code != 2 || !strings.Contains(stderr, "keeps no report")) { + t.Errorf("a report by %s: report = %d, read %v, want %v:\n%s%s", tc.login, code, read, tc.read, stdout, stderr) + } + } +} + +// A report whose ID is not one says so without repeating it as written. +func TestReportErrorsRepeatNothingRaw(t *testing.T) { + s := forged(func(*reportDoc) {}) + // Short enough that a cut to 12 characters keeps the forged line. + raw := strings.Replace(s.raw, `"id":"`+s.doc.ID+`"`, `"id":"\nforged"`, 1) + code, _, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(raw)+"\n")}}, "--pr", "90") + for _, ln := range strings.Split(stderr, "\n") { + if strings.HasPrefix(ln, "forged") { + t.Errorf("report = %d, printed the forged line %q", code, ln) + } + } + if code != 2 { + t.Errorf("report = %d, want 2:\n%s", code, stderr) + } +} diff --git a/cmd/backport-batch/rules.go b/cmd/backport-batch/rules.go index 632fd7ea8..e7aa2740a 100644 --- a/cmd/backport-batch/rules.go +++ b/cmd/backport-batch/rules.go @@ -40,6 +40,9 @@ const ( // Rule 1 for a line that started under the policy after it branched: // merged before that start. ruleBeforeStart = 101 + // Not a rule of the policy: a maintainer dropped the original from the + // line with run --drop. + ruleDropped = 102 ) const ( diff --git a/cmd/backport-batch/shape.go b/cmd/backport-batch/shape.go index 07a5b91f1..ee0a6940d 100644 --- a/cmd/backport-batch/shape.go +++ b/cmd/backport-batch/shape.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "errors" "fmt" "os" "slices" @@ -33,10 +34,11 @@ import ( // pickOriginal picks one original onto HEAD in that shape. A conflict on any // of its commits takes the branch back to where it was before the original, -// so nothing of it is left half applied. Hooks are off so that each commit -// carries its message and -x line as git writes them; signing is left to the -// user's configuration. -func pickOriginal(v *verdict) (pick, error) { +// so nothing of it is left half applied, and so does a clean pick validate +// finds leaving something of the original without effect: partial, with +// those paths. Hooks are off so that each commit carries its message and -x +// line as git writes them; signing is left to the user's configuration. +func pickOriginal(v *verdict, validate func(base, head string) ([]string, error)) (pick, error) { p := pick{v: v} base, err := resolve("HEAD") if err != nil { @@ -59,10 +61,34 @@ func pickOriginal(v *verdict) (pick, error) { switch { case err != nil: return p, err + case head == base && validate != nil: + // Picks that change nothing carry the original only when the line + // already has its changes: a change made where its branch merged + // main is in no pick. + omitted, err := validate(base, head) + if err != nil { + return p, err + } + if len(omitted) > 0 { + p.status, p.files = "partial", omitted + return p, nil + } + p.status = "empty" + return p, nil case head == base: p.status = "empty" return p, nil - case v.o.parents < 2: + } + if validate != nil { + omitted, err := validate(base, head) + if err == nil && len(omitted) > 0 { + p.status, p.files = "partial", omitted + } + if err != nil || len(omitted) > 0 { + return p, errors.Join(err, resetTo(base)) + } + } + if v.o.parents < 2 { p.status, p.commit = "picked", head return p, nil } @@ -301,7 +327,8 @@ func checkMerge(b *batch, u unit) (string, *verdict, error) { } // contained says why the change from base to tip is not inside the files v -// changes, or "" when it is. +// changes, or the paths a rename on main or on the line gave them at base, +// or "" when it is. func contained(b *batch, v *verdict, base, tip string) (string, error) { if err := b.load(v); err != nil { return "", err @@ -313,7 +340,14 @@ func contained(b *batch, v *verdict, base, tip string) (string, error) { if len(changes) == 0 { return fmt.Sprintf("%.12s changes nothing", tip), nil } + corr, _, err := b.correspondence(v, base) + if err != nil { + return "", err + } allowed := paths(v.changes) + for _, c := range corr { + allowed = append(allowed, c) + } var extra []string for _, p := range paths(changes) { if !slices.Contains(allowed, p) { @@ -358,15 +392,50 @@ func replayUnit(u unit) (string, []string, error) { return "", nil, err } -// summarize says, for each original the batch head carries in units, how many -// commits it brought and which of them differ from the commit they were picked -// from: a resolution, or a pick git had to fit to the line. -func summarize(b *batch, head string, on []*verdict) (map[*verdict]string, error) { +// pickPair is a pick on the batch and the commit it was picked from. +type pickPair struct{ source, pick string } + +// unitShape is what an original's unit on the batch brought: how many +// commits, the picks that differ from the commit they came from, a +// resolution or a pick git had to fit to the line, and whether the unit as a +// whole makes another change than the original's merge. +type unitShape struct { + unit string + commits int + pairs []pickPair + net bool +} + +// changed says whether the unit differs from its original, so its release +// note may no longer hold. +func (s unitShape) changed() bool { return len(s.pairs) > 0 || s.net } + +// String is the shape as the body lists it. +func (s unitShape) String() string { + out := "1 commit" + if s.commits != 1 { + out = fmt.Sprintf("%d commits", s.commits) + } + if len(s.pairs) > 0 { + var picks []string + for _, p := range s.pairs { + picks = append(picks, codeSpan(p.pick[:12])) + } + out += ", changed in the pick: " + strings.Join(picks, ", ") + } else if s.net { + out += ", changed as a whole: some of its change is not in the picks" + } + return out +} + +// summarize says, for each original the batch head carries in units, what +// its unit brought. +func summarize(b *batch, head string, on []*verdict) (map[*verdict]unitShape, error) { units, err := batchUnits(head, b.lineHead) if err != nil { return nil, err } - out := map[*verdict]string{} + out := map[*verdict]unitShape{} for _, u := range units { v := b.unitOriginal(u) if v == nil || !slices.Contains(on, v) { @@ -380,7 +449,7 @@ func summarize(b *batch, head string, on []*verdict) (map[*verdict]string, error if err != nil { return nil, err } - var changed []string + s := unitShape{unit: u.commit, commits: len(picks)} for _, c := range picks { source := appendedSource(msgs[c].message) if source == "" { @@ -391,16 +460,20 @@ func summarize(b *batch, head string, on []*verdict) (map[*verdict]string, error return nil, err } if !same { - changed = append(changed, codeSpan(c[:12])) + s.pairs = append(s.pairs, pickPair{source: source, pick: c}) } } - s := "1 commit" - if len(picks) != 1 { - s = fmt.Sprintf("%d commits", len(picks)) + // A pick that came out empty leaves no pick to differ, only a net + // change short of the original's. + mine, err := patchIDRange(v.o.merge+"^1", v.o.merge) + if err != nil { + return nil, err } - if len(changed) > 0 { - s += ", changed in the pick: " + strings.Join(changed, ", ") + theirs, err := patchIDRange(u.parents[0], u.commit) + if err != nil { + return nil, err } + s.net = mine != theirs out[v] = s } return out, nil @@ -417,16 +490,17 @@ func plural(n int, word string) string { // parents, by git patch-id of their diffs without context, so that a pick // that only lands among other lines on the line counts as the same. func samePatch(a, b string) (bool, error) { - pa, err := patchID(a) + pa, err := patchIDRange(a+"^", a) if err != nil { return false, err } - pb, err := patchID(b) + pb, err := patchIDRange(b+"^", b) return pa == pb, err } -func patchID(c string) (string, error) { - diff, err := git("diff", "--no-ext-diff", "--no-textconv", "--unified=0", "--end-of-options", c+"^", c) +// patchIDRange is the patch-id of the change from one revision to another. +func patchIDRange(from, to string) (string, error) { + diff, err := git("diff", "--no-ext-diff", "--no-textconv", "--unified=0", "--end-of-options", from, to) if err != nil { return "", err } diff --git a/cmd/backport-batch/verify.go b/cmd/backport-batch/verify.go index 05023a259..0691df641 100644 --- a/cmd/backport-batch/verify.go +++ b/cmd/backport-batch/verify.go @@ -91,6 +91,15 @@ func (a *app) verify(args []string) int { if why == "" && seen[v] != "" { why = fmt.Sprintf("carries #%d again, which %.12s already carries", v.o.number, seen[v]) } + if why == "" { + omitted, _, err := b.omittedEffects(v, u.parents[0], u.commit) + if err != nil { + return a.fail(err) + } + if len(omitted) > 0 { + why = leftOut(v, l, omitted) + } + } if why != "" { refused = append(refused, why) fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why) diff --git a/cmd/backport-batch/wait_test.go b/cmd/backport-batch/wait_test.go new file mode 100644 index 000000000..35dd5c1d6 --- /dev/null +++ b/cmd/backport-batch/wait_test.go @@ -0,0 +1,236 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "strings" + "testing" +) + +// chainWorld has #5 conflict on the line and change a shared line, and #7 +// change that line again: #7 builds on #5. +func chainWorld(t *testing.T) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: five and shared", "", map[string]string{"app/five.txt": "old\n", "app/shared.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): five, the line's way", map[string]string{"app/five.txt": "line\n"}) + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "five\n", "app/shared.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/shared.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + return f, mc5 +} + +// An original that builds on a deferred conflict waits for it: check warns, +// notify lists it after the conflict, and once the conflict lands, the next +// run picks it. +func TestWaitForADeferredDependency(t *testing.T) { + f, mc5 := chainWorld(t) + gh := &fakeGH{} + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + if !strings.Contains(stdout, `{"original":7,"files":[],"needs":[5]}`) || !strings.Contains(stdout, "### Waiting on another original") { + t.Errorf("#7 does not wait on #5:\n%s", stdout) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "warning: #7 waits on #5, a deferred conflict on #90: https://github.com/cozystack/cozystack/pull/7 ") { + t.Errorf("check = %d, want a warning that #7 waits on #5:\n%s\n%s", code, stdout, stderr) + } + code, stdout, _ = f.exec(gh, "notify", "--line", "release-1.1", "--push") + if code != 0 || !strings.Contains(gh.issueComments[len(gh.issueComments)-1].Body, "- [#7](https://github.com/cozystack/cozystack/pull/7), after [#5](https://github.com/cozystack/cozystack/pull/5)") { + t.Errorf("notify = %d, want #7 listed after #5:\n%s\n%v", code, stdout, gh.issueComments) + } + // The batch merges, and #5 lands on the line by hand, resolved: the + // next batch picks #7. + mergeBatch(f, gh) + f.onLine("release-1.1", "fix(app): five (#5)\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "five\n", "app/shared.txt": strings.Replace(appFile, "two", "TWO", 1)}) + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--continue") + if code != 0 || !strings.Contains(stdout, `"picked":[7]`) { + t.Errorf("run --continue after #5 landed = %d, want #7 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// Of several prerequisites still to come, the strongest decides: one not on +// the line at all, then one waiting for a maintainer, then a conflict or a +// deferral, then one rule 6 holds; a chain inherits what its end decides. +func TestWaitReduction(t *testing.T) { + t.Run("a conflict and a workflow wait", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/five.txt": "old\n", "app/a.txt": appFile, "app/b.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): five, the line's way", map[string]string{"app/five.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "five\n", "app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(ci): six", "", map[string]string{".github/workflows/six.yaml": "on: push\n", "app/b.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh := &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 || !strings.Contains(stdout, `{"original":7,"files":[],"needs":[6]}`) { + t.Fatalf("run = %d, want #7 waiting on #6 alone:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#7 waits on #6, which changes .github/workflows/ and waits on batch PR #90 for a maintainer") { + t.Errorf("check --cut = %d, want #7 to hold the cut:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, _ := f.exec(gh, "notify", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, "2 originals on #90 change .github/workflows/ or wait on one that does") { + t.Errorf("notify = %d, want #7 among the waits:\n%s", code, stdout) + } + }) + t.Run("a conflict and one not on the line", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/five.txt": "old\n", "app/a.txt": appFile, "app/b.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): five, the line's way", map[string]string{"app/five.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "five\n", "app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + f.pr(9, "fix(app): nine", "", map[string]string{"app/b.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(10, "fix(app): ten", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/10 fix(app): ten (it waits on #9, which is not on the line either)") { + t.Errorf("check = %d, want #10 to fail it, waiting on #9:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a prerequisite on the open batch", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + gh := &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 || !strings.Contains(stdout, `"picked":[5,7]`) { + t.Fatalf("run = %d, want #5 and #7 picked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 is only on open batch PR #90") || strings.Contains(stdout+stderr, "waits on") { + t.Errorf("check --cut = %d, want #5 and #7 only on the batch and no wait:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a direct reason", func(t *testing.T) { + f := featWorld(t) + gh := &fakeGH{} + openBatch(t, f, gh) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "needs the tap", "--push"); code != 0 { + t.Fatalf("run --defer = %d\n%s", code, stderr) + } + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "warning: deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check = %d, want #5 deferred for its reason, not parked:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a prerequisite rule 6 holds", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + mc7 := f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/seven.txt": "7\n"}) + f.onLine("release-1.1", "fix(app): seven (#7)\n\n(cherry picked from commit "+mc7+")", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/seven.txt": "7\n"}) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.0"); code != 0 || !strings.Contains(stderr, "note: #7 waits on #5, which is not on release-1.1 yet") { + t.Errorf("check of release-1.0 = %d, want a note that #7 waits on #5:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a chain", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(4, "fix(app): four", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!!", 1)}) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/7 fix(app): seven (it waits on #5, which is not on the line either)") { + t.Errorf("check = %d, want #7 failing it through #5:\n%s\n%s", code, stdout, stderr) + } + }) +} + +// run's report, check and notify give an original one disposition: a +// workflow original whose only missing dependency is a test waits for a +// maintainer, and one deferred for a reason stays deferred, though it +// builds on what never comes; neither is parked. +func TestDispositionOrderIsSharedByRunCheckNotify(t *testing.T) { + f := auxWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(7, "fix(app): fix the tap", "", map[string]string{"app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "7", "--reason", "waits for the tap", "--push") + if code != 0 { + t.Fatalf("run --defer = %d\n%s", code, stderr) + } + if strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `{"original":5,"files":[".github/workflows/release.yaml"],"workflows":true}`) || !strings.Contains(stdout, `{"original":7,"files":[],"reason":"waits for the tap"}`) { + t.Errorf("the report does not have #5 waiting and #7 deferred, unparked:\n%s", stdout) + } + reopen(f, gh) + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || strings.Contains(stderr, "parked") || !strings.Contains(stderr, "warning: waiting on #90 for a maintainer, since it changes .github/workflows/: https://github.com/cozystack/cozystack/pull/5 ") || !strings.Contains(stderr, "warning: deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/7 ") { + t.Errorf("check = %d, want #5 waiting and #7 deferred:\n%s\n%s", code, stdout, stderr) + } + code, stdout, _ = f.exec(gh, "notify", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, "1 originals on #90 change .github/workflows/") || strings.Contains(stdout, "parked") || strings.Contains(stdout, "conflicts") { + t.Errorf("notify = %d, want #5 waiting and nothing for #7:\n%s", code, stdout) + } +} + +// A never and a pending prerequisite together park. +func TestParkWinsOverWait(t *testing.T) { + f, _ := chainWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(8, "fix(app): eight", "", map[string]string{"app/shared.txt": strings.Replace(appFile, "two", "TWO!!", 1), "app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"kind":"dependency","needs":[20]`) || strings.Contains(stdout, `"original":8,"files":[],"needs"`) { + t.Errorf("run = %d, want #8 parked on #20, not waiting on #7:\n%s\n%s", code, stdout, stderr) + } +} + +// A prerequisite picked earlier in the same batch satisfies what builds on +// it, and so does one whose pick came out empty because an earlier pick +// already carried it: #4280 and #3938. +func TestDependencySatisfiedEarlierInTheBatch(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(3, "fix(kafka): delete hooks", "", map[string]string{"app/hooks.yaml": appFile}) + f.pr(4, "fix(kafka): fix the delete hook", "", map[string]string{"app/hooks.yaml": strings.Replace(appFile, "two", "TWO", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"picked":[3,4]`) { + t.Errorf("run = %d, want #3 and #4 picked:\n%s\n%s", code, stdout, stderr) + } + + // #30 adds back the file #2 brought and a feat took away, so its pick + // comes out empty after #2's, and #4 builds on #30. + f = newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(20, "fix(kafka): delete hooks, and two", "", map[string]string{"app/hooks.yaml": appFile, "app/two.txt": "2\n"}) + f.pr(21, "feat(kafka): no delete hooks", "", map[string]string{"app/hooks.yaml": ""}) + f.pr(30, "fix(kafka): delete hooks again", "", map[string]string{"app/hooks.yaml": appFile}) + f.pr(4, "fix(kafka): fix the delete hook", "", map[string]string{"app/hooks.yaml": strings.Replace(appFile, "two", "TWO", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"picked":[4,20]`) || strings.Contains(stdout, `"needs"`) { + t.Errorf("run with #30's pick empty = %d, want #4 picked:\n%s\n%s", code, stdout, stderr) + } +} diff --git a/cmd/backport-batch/workflows_test.go b/cmd/backport-batch/workflows_test.go index 69f402974..97547f6b5 100644 --- a/cmd/backport-batch/workflows_test.go +++ b/cmd/backport-batch/workflows_test.go @@ -110,7 +110,8 @@ func TestWorkflowChangesWaitUntilOnTheLine(t *testing.T) { if code != 0 || !strings.Contains(stdout, "1 originals on #90 change .github/workflows/ and wait for a maintainer") || strings.Contains(stdout, "deferred conflicts") { t.Errorf("notify = %d, %q; want #5 for a maintainer and no conflict\n%s", code, stdout, stderr) } - if len(gh.issueComments) != 1 || !strings.Contains(gh.issueComments[0].Body, "[#5](") || !strings.Contains(gh.issueComments[0].Body, "#changes-to-workflows") || strings.Contains(gh.issueComments[0].Body, "These conflicts") { + notes := slices.DeleteFunc(slices.Clone(gh.issueComments), func(c issueComment) bool { return !strings.HasPrefix(c.Body, notifyMarker) }) + if len(notes) != 1 || !strings.Contains(notes[0].Body, "[#5](") || !strings.Contains(notes[0].Body, "#changes-to-workflows") || strings.Contains(notes[0].Body, "These conflicts") { t.Errorf("the comment does not list #5 for a maintainer: %+v", gh.issueComments) } @@ -173,8 +174,12 @@ func TestDeferKeepsWorkflowChangesWaiting(t *testing.T) { } f.git("switch", "-q", "main") reopen(f, gh) - if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); strings.Contains(stdout, "#5 changes") { - t.Errorf("check --cut = %d, %q; a deferred #5 still stops the cut as waiting", code, stdout) + // #6 and #7 are still only on the batch, which fails the cut; #5 does not. + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || strings.Contains(stdout, "#5 changes") || !strings.Contains(stdout, "#6 is only on open batch PR #90") { + t.Errorf("check --cut = %d, %q; want 1 for #6 and #7 alone, a deferred #5 no longer waiting", code, stdout) + } + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ") { + t.Errorf("check = %d, want 0 with a warning about #5:\n%s", code, stderr) } } diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go new file mode 100644 index 000000000..558908671 --- /dev/null +++ b/cmd/observability-proxy/main.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command observability-proxy serves tenant logs from the VictoriaLogs +// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the +// tenant's namespace scope forced into every upstream request. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "os" + "os/signal" + "sync/atomic" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + + "github.com/cozystack/cozystack/internal/observabilityproxy" +) + +type options struct { + listenAddress string + probeAddress string + clusterDomain string + rootTenant string + kubeconfig string + authCacheTTL time.Duration + requestTimeout time.Duration + tailIdleTimeout time.Duration + shutdownTimeout time.Duration + logs observabilityproxy.LogsLimits + limiter observabilityproxy.LimiterConfig +} + +func parseFlags(args []string) (options, error) { + var o options + fs := flag.NewFlagSet("observability-proxy", flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.StringVar(&o.listenAddress, "listen-address", ":8080", "address of the API listener") + fs.StringVar(&o.probeAddress, "probe-address", ":8081", "address serving /healthz, /readyz and /metrics") + fs.StringVar(&o.clusterDomain, "cluster-domain", "cozy.local", "cluster DNS domain used to reach the stores") + fs.StringVar(&o.rootTenant, "root-tenant", "tenant-root", "tenant namespace hosting the default store") + fs.StringVar(&o.kubeconfig, "kubeconfig", "", "kubeconfig path; in-cluster config when empty") + fs.DurationVar(&o.authCacheTTL, "auth-cache-ttl", 10*time.Second, "how long TokenReview and SubjectAccessReview answers are cached") + fs.DurationVar(&o.requestTimeout, "request-timeout", time.Minute, "deadline for reading the request body, the token and access reviews, and every upstream endpoint but tail; 0 disables it") + fs.DurationVar(&o.tailIdleTimeout, "tail-idle-timeout", 5*time.Minute, "close a tail after this long without a line") + fs.DurationVar(&o.shutdownTimeout, "shutdown-timeout", 15*time.Second, "grace period for in-flight requests on shutdown") + fs.DurationVar(&o.logs.DefaultRange, "logs-default-range", time.Hour, "window used when a logs request has no start") + // One week covers an incident investigation and keeps a single query + // to a fraction of the default one-month log retention. + fs.DurationVar(&o.logs.MaxRange, "logs-max-range", 7*24*time.Hour, "longest [start, end] a logs request may span; wider windows are clamped") + // Prometheus's cap on points per series in a range query. + fs.IntVar(&o.logs.MaxBuckets, "logs-max-buckets", 11000, "most buckets hits and stats_query_range may return over a window") + fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit") + fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set") + fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset") + fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") + fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") + fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") + fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") + fs.IntVar(&o.limiter.MaxTails, "max-tails", 200, "open tails in total; 0 disables") + if err := fs.Parse(args); err != nil { + return o, err + } + // Parsing stops at the first non-flag, so every flag after it would + // be dropped silently. + if fs.NArg() > 0 { + return o, fmt.Errorf("unexpected argument %q", fs.Arg(0)) + } + switch { + case o.rootTenant == "": + return o, errors.New("--root-tenant must not be empty") + case o.clusterDomain == "": + return o, errors.New("--cluster-domain must not be empty") + case o.logs.DefaultRange <= 0 || o.logs.MaxRange < o.logs.DefaultRange: + return o, errors.New("--logs-default-range must be positive and not above --logs-max-range") + // Below four buckets (three steps once one is held back for + // alignment), or a range under three seconds, the step cap that keeps + // vlselect's step alignment inside the range cannot hold. + case o.logs.MaxBuckets < 4: + return o, errors.New("--logs-max-buckets must be at least 4") + case o.logs.MaxRange < 3*time.Second: + return o, errors.New("--logs-max-range must be at least 3s") + case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit: + return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit") + case o.logs.MaxTailResume < 0: + return o, errors.New("--logs-max-tail-resume must not be negative") + case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0: + return o, errors.New("timeouts and the auth cache TTL must not be negative") + // NaN passes every comparison and +Inf never empties a bucket; either + // would switch the rate limit off without saying so. + case math.IsNaN(o.limiter.RatePerUser) || math.IsInf(o.limiter.RatePerUser, 0): + return o, errors.New("--rate-limit-per-user must be a finite number") + case o.limiter.RatePerUser < 0 || o.limiter.RateBurst < 0 || o.limiter.MaxConcurrentPerUser < 0 || + o.limiter.MaxTailsPerUser < 0 || o.limiter.MaxTails < 0: + return o, errors.New("limits must not be negative; 0 disables a limit") + } + return o, nil +} + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil))) + o, err := parseFlags(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "observability-proxy: %v\n", err) + os.Exit(2) + } + if err := run(o); err != nil { + slog.Error("exiting", "error", err) + os.Exit(1) + } +} + +func run(o options) error { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + restConfig, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) + if err != nil { + return fmt.Errorf("loading kubeconfig: %w", err) + } + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("building client: %w", err) + } + + factory := informers.NewSharedInformerFactory(client, 0) + namespaces := factory.Core().V1().Namespaces() + lister := namespaces.Lister() + factory.Start(ctx.Done()) + var ready atomic.Bool + go func() { + if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) { + ready.Store(true) + } + }() + + reg := prometheus.NewRegistry() + reg.MustRegister(collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + proxy := observabilityproxy.NewServer(observabilityproxy.Config{ + ClusterDomain: o.clusterDomain, + Logs: o.logs, + Limiter: o.limiter, + RequestTimeout: o.requestTimeout, + TailIdleTimeout: o.tailIdleTimeout, + }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg) + + probes := http.NewServeMux() + probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) + probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + if !ready.Load() { + http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(w, "ok") + }) + probes.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + + // No WriteTimeout on the API listener: it would cut live tails, which + // the handler bounds with its own idle timeout instead. + servers := []*http.Server{ + {Addr: o.listenAddress, Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute}, + {Addr: o.probeAddress, Handler: probes, ReadHeaderTimeout: 10 * time.Second}, + } + errs := make(chan error, len(servers)) + for _, srv := range servers { + go func() { + slog.Info("listening", "address", srv.Addr) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + errs <- err + } + }() + } + + var runErr error + select { + case <-ctx.Done(): + case runErr = <-errs: + } + stop() + shutdownCtx, cancel := context.WithTimeout(context.Background(), o.shutdownTimeout) + defer cancel() + for _, srv := range servers { + if err := srv.Shutdown(shutdownCtx); err != nil { + // Open tails never go idle on their own; cut them so clients + // reconnect with since to another replica. + _ = srv.Close() + } + } + return runErr +} diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go new file mode 100644 index 000000000..ea678461c --- /dev/null +++ b/cmd/observability-proxy/main_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + "time" +) + +func TestParseFlagsDefaults(t *testing.T) { + o, err := parseFlags(nil) + if err != nil { + t.Fatal(err) + } + if o.listenAddress != ":8080" || o.probeAddress != ":8081" || o.clusterDomain != "cozy.local" || o.rootTenant != "tenant-root" { + t.Errorf("contract defaults changed: %+v", o) + } + if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour { + t.Errorf("limit defaults changed: %+v", o) + } +} + +func TestParseFlagsOverrides(t *testing.T) { + o, err := parseFlags([]string{"--cluster-domain=example.org", "--logs-max-range=24h", "--max-tails=3"}) + if err != nil { + t.Fatal(err) + } + if o.clusterDomain != "example.org" || o.logs.MaxRange != 24*time.Hour || o.limiter.MaxTails != 3 { + t.Errorf("overrides not applied: %+v", o) + } + // The smallest bucket count and range the step cap can satisfy. + if _, err := parseFlags([]string{"--logs-max-buckets=4", "--logs-max-range=3s", "--logs-default-range=1s"}); err != nil { + t.Errorf("smallest satisfiable step cap refused: %v", err) + } +} + +func TestParseFlagsInvalid(t *testing.T) { + for _, args := range [][]string{ + {"--root-tenant="}, + {"--cluster-domain="}, + {"--logs-default-range=48h", "--logs-max-range=24h"}, + {"--logs-max-buckets=3"}, + {"--logs-max-range=2s", "--logs-default-range=1s"}, + {"--shutdown-timeout=-1s"}, + {"--logs-default-limit=0"}, + {"--logs-default-limit=100", "--logs-max-limit=10"}, + {"--logs-max-tail-resume=-1m"}, + {"--request-timeout=-1s"}, + {"--tail-idle-timeout=-1s"}, + {"--auth-cache-ttl=-1s"}, + {"--rate-limit-per-user=-1"}, + {"--rate-limit-per-user=NaN"}, + {"--rate-limit-per-user=+Inf"}, + {"--rate-limit-burst=-1"}, + {"--max-concurrent-per-user=-1"}, + {"--max-tails-per-user=-1"}, + {"--max-tails=-1"}, + {"--unknown"}, + {"unexpected", "--root-tenant=tenant-x"}, + } { + if _, err := parseFlags(args); err == nil { + t.Errorf("parseFlags(%v) accepted", args) + } + } +} diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index 88b86edbd..cfed8704f 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -134,7 +134,7 @@ git log v0.37.1..v0.37.2 --pretty=format:"%h - %s (%an, %ar)" ``` **⚠️ IMPORTANT: Check for backports.** They reach a release line in two shapes: -- A backport batch ([`docs/release/backports.md`](../release/backports.md)): a merge commit titled `fix(backport): batch for release-X.Y, (#BATCH_PR)`. Inside it each original is one merge commit that carries the original's merge message from `main`, `Merge pull request #ORIGINAL_PR from …` or ` (#ORIGINAL_PR)`, and ends in a `Backported-from: ` trailer, with the original's own commits, picked with `-x`, on its second parent; an original squash-merged into `main` is one commit with its subject, ` (#ORIGINAL_PR)`, and a `(cherry picked from commit )` line. Read the original's number, title and author from those, list the batch with `git log --first-parent` inside it, and never write an entry for the batch PR itself or for a picked commit of an original. +- A backport batch ([`docs/release/backports.md`](../release/backports.md)): a merge commit titled `fix(backport): batch for release-X.Y, (#BATCH_PR)`. Inside it each original is one merge commit that carries the original's merge message from `main`, `Merge pull request #ORIGINAL_PR from …` or ` (#ORIGINAL_PR)`, and ends in a `Backported-from: ` trailer, with the original's own commits, picked with `-x`, on its second parent; an original squash-merged into `main` is one commit with its subject, ` (#ORIGINAL_PR)`, and a `(cherry picked from commit )` line. Read the original's number, title and author from those, list the batch with `git log --first-parent` inside it, and never write an entry for the batch PR itself or for a picked commit of an original. An original whose pick differs from it is listed under "Check these notes" in the batch merge's message, which carries the batch PR's body, and in its record's `changed` list: write its entry from what its pick changed on the line, read with the `git range-diff` and `git diff` commands listed there, not from the original's release note. - A single backport PR, from before the batch (the retired label-driven bot) or a release-only fix: commits with "[Backport release-X.Y]" in the message, whose original PR number is in the backport commit message or PR description. - Use the original PR author (not the backport or batch PR author) when creating changelog entries - **MUST** combine the original and backport into a **single entry** with both PR numbers (e.g. `#1606, backport #1609`, where a batch's original reads `#1606, backport #BATCH_PR`). Do NOT list the original and the backport as two separate entries — that produces duplicates in the changelog. diff --git a/docs/release/backport-mechanics.md b/docs/release/backport-mechanics.md index f713d20aa..3474bffea 100644 --- a/docs/release/backport-mechanics.md +++ b/docs/release/backport-mechanics.md @@ -14,7 +14,7 @@ go build ./cmd/backport-batch ./backport-batch check --line release-X.Y --cut ``` -[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check. A change that reached the line by hand counts like any other, by what reached the line. +[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check, also once that batch merged or closed. An original that waits on the open batch for a maintainer fails the cut, and so does a unit on the open batch that carries only part of its original. An original parked because it builds on a change the line never gets, or would land only in part, prints as a note; one that waits on another original fails, warns or notes by what it waits on. A change that reached the line by hand counts like any other, by what reached the line. ### Backporting code that touches monotonic counters diff --git a/docs/release/backport-tasks/README.md b/docs/release/backport-tasks/README.md index eba2c3738..dbbb536ad 100644 --- a/docs/release/backport-tasks/README.md +++ b/docs/release/backport-tasks/README.md @@ -10,9 +10,9 @@ The [backport batch](../backports.md) is built by [`backport-batch`](../../../cm ### Rules for every task - Push nothing but the branch of the open batch pull request, `backport-batch/-` in this repository, and only what the task's check accepted. Never push to `main` or a `release-*` branch, and never open a second batch pull request: the scheduled run continues the open one. -- Change no file the original pull request does not change. A conflict that cannot be resolved inside those files is not resolved: the original stays deferred. +- Change no file the original pull request does not change, or the path a rename on `main` or on the line gave it. A conflict that cannot be resolved inside those files is not resolved: the original stays deferred. - Never rewrite the batch branch by hand. A plain push to it is a fast-forward; only `backport-batch run --defer --push` replaces it, leased on the head it rebuilt. -- Leave the batch pull request's title and body to the tool: it rewrites both on every run, and a hand edit is lost or, inside the `backport-batch` block, makes `check` refuse the body. +- Leave the batch pull request's title and body to the tool: it rewrites both on every run, and a hand edit is lost or, inside the `backport-batch` block, makes `run`, `check`, `fold` and `notify` refuse the body, and inside the `backport-batch-report` block makes `run --push` and `backport-batch report` refuse it. Leave the tool's report comments as they are too: each keeps one classification of the line, and an edited one makes `backport-batch report` refuse it. - When the task cannot be done within these rules, push nothing and leave a comment on the batch pull request saying why, so the next maintainer to look finds it. Nothing is lost by waiting: a red batch does not merge, and a deferred original goes to the next batch. ### Getting the tool @@ -26,4 +26,4 @@ cd ../backport-task go build -o /tmp/backport-batch ./cmd/backport-batch ``` -Run it from that worktree. Every command but `notify` fetches `origin` first, and `run`, `check`, `fold` and `lines` read the policy from `origin/main`. +Run it from that worktree. Every command but `notify` and `report` fetches `origin` first, and `run`, `check`, `fold` and `lines` read the policy from `origin/main`. diff --git a/docs/release/backport-tasks/failed-ci.md b/docs/release/backport-tasks/failed-ci.md index e845eff99..18b26f4dc 100644 --- a/docs/release/backport-tasks/failed-ci.md +++ b/docs/release/backport-tasks/failed-ci.md @@ -33,6 +33,8 @@ It rebuilds the branch locally without those originals and prints the new body, It replaces the batch branch only if nobody pushed onto it since the rebuild read it, and updates the pull request body. CI runs again on the new head; if it fails again on another original, repeat from step 1. +When the culprits are everything the batch carries, there is no branch left to push: the body printed without `--push` says the batch is closing, and `--push` records the deferrals in the batch pull request, closes it and deletes its branch, still only if nobody pushed onto it since. The next batch tries them again. If it stops part way, the next `run --push` on the line finishes the close, and says so. + ### Never - Defer an original the failure does not reach, to make the batch green by taking out more than the culprit. diff --git a/docs/release/backport-tasks/resolve-conflict.md b/docs/release/backport-tasks/resolve-conflict.md index 928bc8966..885e568d4 100644 --- a/docs/release/backport-tasks/resolve-conflict.md +++ b/docs/release/backport-tasks/resolve-conflict.md @@ -4,7 +4,7 @@ The batch carries a pull request merged into `main` as its own commits, each pic ### Inputs -- The open batch pull request for the line, its branch `backport-batch/-` and its head commit, as GitHub shows them: `gh pr view --json headRefName,headRefOid`. +- The open batch pull request for the line, its branch `backport-batch/-` and its head commit, as GitHub shows them: `gh pr view --json headRefName,headRefOid`. The resolution lands on that batch, so without an open batch pull request there is nothing to resolve onto: wait for the next batch, which lists the conflict again, or port the change as a pull request of its own against the line, as [fixes that exist only on a release line](../backports.md#fixes-that-exist-only-on-a-release-line) describes, and drop the original from the line. - The original's number `N` from the **Deferred conflicts** list, and its merge commit on `main`: `gh pr view N --json mergeCommit --jq .mergeCommit.oid`. It has two parents when the pull request was merged, one when it was squashed. - For a merged pull request, its own commits, oldest first, without the merges that brought `main` into its branch: `git rev-list --reverse --topo-order --no-merges ^1..^2`. - The files the original changes, `git diff --name-only ^1 `, which are the only files the resolution may touch. @@ -34,7 +34,7 @@ git add git -c core.hooksPath=/dev/null cherry-pick --continue --no-edit ``` -When the original builds on something the line does not have, a helper, a value or another pull request, the resolution would have to bring that along, which this task does not allow: abort with `git cherry-pick --abort`, go back with `git switch --detach "$base"`, and comment on the batch pull request which change the original depends on; it stays deferred until that change is on the line. A commit whose change the line already has stops the pick as empty; skip it with `git cherry-pick --skip`. +When the original builds on something the line does not have, a helper, a value or another pull request, the resolution would have to bring that along, which this task does not allow: abort with `git cherry-pick --abort`, go back with `git switch --detach "$base"`, and comment on the batch pull request which change the original depends on; it stays deferred until that change is on the line. An original the batch parked because it builds on such a change is the same task, and lands only as a complete adaptation: every file it changes changed, unless the line already has that file as the original makes it. When that change will never reach the line, drop the original there instead, with `backport-batch run --line --drop N --reason "" --push`, so later batches stop trying it. A commit whose change the line already has stops the pick as empty; skip it with `git cherry-pick --skip`. A resolution that ports only what the line has and leaves a file of the original as it was, one the line does not have for instance, is refused by `fold`: such a port goes to the line as a pull request of its own, as [fixes that exist only on a release line](../backports.md#fixes-that-exist-only-on-a-release-line) describes, and the original is dropped there. For a merged pull request, join the picks to the batch head with the merge commit: diff --git a/docs/release/backports.md b/docs/release/backports.md index c613cc980..ad7424c40 100644 --- a/docs/release/backports.md +++ b/docs/release/backports.md @@ -21,6 +21,14 @@ The batch takes a pull request from `main` into release line X.Y when all of the The batch reads the pull requests off `main`'s first-parent history, one per merge commit or squash commit that names its pull request. A commit there that names none, a direct push for instance, and a merge of more than two parents are never taken; the batch pull request counts them. +### What the batch leaves out on one line + +A pull request the rules take for every line can still be wrong for one of them, for instance because it builds on code that line does not have. A maintainer drops it from that line with `./backport-batch run --line --drop --reason --push`: the batch takes it off, if it carries it, and no later batch for that line picks it again. The drop is kept in the [line's record](#the-lines-record-and-reports), so it needs an open batch pull request to record it on; before a line's first batch pull request exists there is nothing to record it on, and the command refuses. `--undrop ` withdraws it, and the next batch tries the pull request again. A pull request the line already carries cannot be dropped. To land a hand-made port instead, open it as a pull request of its own, as [fixes that exist only on a release line](#fixes-that-exist-only-on-a-release-line) describes, and drop the original. + +The batch also leaves a pull request out of a line on its own when it cannot land there whole. A pick that would leave part of what the pull request does on `main` without effect on the line, a change to a file the line does not have, or the deletion of a file the line never had or already removed, is not made: the pull request is parked, the batch pull request lists it under "Parked" with those files, and `check` prints it as a note without failing. A file the line already has as the pull request makes it counts as there, and so does a file `main` or the line renamed, under its other name. A unit already on the open batch that does only part of its original fails `check --cut` and `backport-batch verify`, whatever its label says, until `backport-batch run --defer` takes it out. What applies of a parked pull request goes to the line as a pull request of its own, as [fixes that exist only on a release line](#fixes-that-exist-only-on-a-release-line) describes. + +A pull request is parked too when it builds on a change the line never gets: it changes a file a `feat` added on `main`, say, or a line such a change wrote, and the line has neither. `backport-batch` reads that from git, as [its README](../../cmd/backport-batch/README.md#what-an-original-builds-on) describes, and names the change in the batch pull request. One that builds on a change that is still to come, a deferred conflict, one waiting for a maintainer, one rule 6 holds or one not tried yet, waits for it instead, listed under "Waiting on another original", and the batch tries it again once that change is on the line; one picked earlier in the same batch counts as there. A complete hand adaptation of a parked pull request, every file it changes changed, still lands on the batch through [resolving a conflict](./backport-tasks/resolve-conflict.md). + ### What waits for the next minor `feat`, `chore` and `docs` pull requests, and dependency bumps (scope `deps`), ship with the next minor, not in a patch release; only a maintainer's `backport/include`, below, takes one. If a class of fixes should also wait for a minor, it gets a new rule on this page; a single pull request that the rules get wrong is what the overrides below are for. @@ -29,9 +37,11 @@ The batch reads the pull requests off `main`'s first-parent history, one per mer A maintainer overrides the rules for one pull request with a label on the merged pull request on `main`: -- `backport/include` takes it although rules 2 to 4 keep it out, a `feat` for instance, or although it merged before its line started under this policy. It still has to pick cleanly or have its conflict resolved, pass CI on the batch, and, on an older line, already be on the newer one, and a line that already carries it does not take it again. +- `backport/include` takes it although rules 2 to 4 keep it out, a `feat` for instance, or although it merged before its line started under this policy. It still has to pick cleanly or have its conflict resolved, pass CI on the batch, and, on an older line, already be on the newer one, and a line that already carries it does not take it again. It lifts no park, as [what the batch leaves out on one line](#what-the-batch-leaves-out-on-one-line) describes. - `backport/skip` keeps it out although the rules take it, and `backport-batch check` stops asking for it. An open batch that picked it before the label came keeps it until `backport-batch run --defer` takes it out; the batch pull request says so meanwhile. +A label decides for every line. To keep a pull request out of one line only, drop it there with `backport-batch run --drop`, as [what the batch leaves out on one line](#what-the-batch-leaves-out-on-one-line) describes; `backport/include` does not lift a drop. + `backport/skip` wins when a pull request carries both. Each run reads the labels afresh, and the batch pull request lists every override and what became of it. ### Which lines the batch runs for @@ -59,7 +69,7 @@ released-lines: 2 ### Fixes that exist only on a release line -Some fixes go to a release line as a pull request of their own. Such pull requests are occasional, and maintainers open them: for a fix with no counterpart on `main`, for one that on `main` came as part of another pull request, or for one written for `main` rather than for the line. When the counterpart on `main` is a pull request of its own that the batch would take, the batch skips it as already on the line once picking it would change nothing there; otherwise a maintainer labels it `backport/skip`. +Some fixes go to a release line as a pull request of their own. Such pull requests are occasional, and maintainers open them: for a fix with no counterpart on `main`, for one that on `main` came as part of another pull request, or for one written for `main` rather than for the line. When the counterpart on `main` is a pull request of its own that the batch would take, the batch skips it as already on the line once picking it would change nothing there; otherwise a maintainer labels it `backport/skip`. A port of only what a line has of a pull request from `main`, one that leaves out what the line lacks, is such a pull request as well: the batch carries no unit that does part of its original, and `backport-batch fold` refuses one. A pull request opened against a `release-*` branch is closed automatically, with a comment that links to this page, when its author is a user account, not a bot, that GitHub names neither the repository's owner nor a member nor a collaborator, and whose access to the repository GitHub reports as read or none. Anyone with triage access or more keeps their pull request open. When GitHub's answer on the author's access cannot be read, the check closes nothing and fails, naming the read that failed. The check runs only when a pull request is opened, so a maintainer keeps a contributor's release-only fix by reopening it. [`hack/close-manual-backport.sh`](../../hack/close-manual-backport.sh) does this from [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml); run it with `--pr --dry-run` to see what it would decide for a given pull request when it is opened. @@ -81,15 +91,21 @@ A run by hand trusts the open batch pull request the scheduled run opened as `co `./backport-batch lines` prints the lines, newest first. `run`, `check`, `fold` and `lines` read the policy block from this page on `main`, so until the page with its block is on `main` they need `--policy ` naming a ref that has it, and the scheduled run, which passes none, fails. -`run` picks what the rules take onto `backport-batch/-` and opens a draft pull request against the line. A pull request merged into `main` with a merge commit goes onto the batch as its own commits, oldest first, each picked with `git cherry-pick -x --signoff` so it keeps its author, its message and its sign-offs and gains the sign-off of whoever runs the batch; commits that merged `main` into the pull request's branch are left out. A merge commit joins them to the batch: it carries the original's merge message from `main` and ends in a `Backported-from: ` trailer. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, nothing of that original stays on the batch. The batch pull request is merged into the line with a merge commit, so all of this reaches the line as it is. Its body lists the originals with how many commits each brought and which of them differ from the commit they came from, the conflicts it deferred with the commit that failed and its files, and what it skipped under which rule. While that pull request is open, the next run continues it, `--continue` reading its head, rather than opening another. CI on the batch pull request runs the line's build and tests, and a maintainer merges it. Since rule 6 reads the newer line itself, an older line takes a fix in the batch after the newer line's batch with that fix has merged. +`run` picks what the rules take onto `backport-batch/-` and opens a draft pull request against the line. A pull request merged into `main` with a merge commit goes onto the batch as its own commits, oldest first, each picked with `git cherry-pick -x --signoff` so it keeps its author, its message and its sign-offs and gains the sign-off of whoever runs the batch; commits that merged `main` into the pull request's branch are left out. A merge commit joins them to the batch: it carries the original's merge message from `main` and ends in a `Backported-from: ` trailer. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, nothing of that original stays on the batch. The batch pull request is merged into the line with a merge commit, so all of this reaches the line as it is. Its body lists the originals with how many commits each brought and which of them differ from the commit they came from, whose release notes wait under "Check these notes" with what to compare, since a note may not hold for a pick that differs, the conflicts it deferred with the commit that failed and its files, what it parked and why, and what it skipped under which rule, and ends with the [line's record and the run's report](#the-lines-record-and-reports); `--push` prints the pull request's URL. While that pull request is open, the next run continues it, `--continue` reading its head, rather than opening another. CI on the batch pull request runs the line's build and tests, and a maintainer merges it. Since rule 6 reads the newer line itself, an older line takes a fix in the batch after the newer line's batch with that fix has merged. -A deferred conflict waits for the next batch. To land it sooner, pick the original's commits onto the batch head with `git cherry-pick -x`, resolving each conflict, and join them to the batch head with a merge commit that names the original; an original squash-merged into `main` is one pick of its commit there, with no merge. Check that merge or that pick with `./backport-batch fold --line --original --commit `, and push it onto the batch branch, as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes. CI runs `backport-batch verify` on every push to a batch pull request, which holds every original on it to the same shape. When CI on the batch pull request fails, the original it comes from is taken out with `./backport-batch run --line --defer --reason --push` and goes to the next batch, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the rest of the batch merges without it. +A deferred conflict waits for the next batch. To land it sooner, pick the original's commits onto the batch head with `git cherry-pick -x`, resolving each conflict, and join them to the batch head with a merge commit that names the original; an original squash-merged into `main` is one pick of its commit there, with no merge. Check that merge or that pick with `./backport-batch fold --line --original --commit `, and push it onto the batch branch, as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes. CI runs `backport-batch verify` on every push to a batch pull request, which holds every original on it to the same shape. When CI on the batch pull request fails, the original it comes from is taken out with `./backport-batch run --line --defer --reason --push` and goes to the next batch, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the rest of the batch merges without it. When it is all the batch carries, `--push` records why in the batch pull request, closes it and deletes its branch, and the next batch tries it again. `--drop` and `--undrop` change the record of the open batch pull request in the same way, and need one. -Before a patch release or an rc is cut from a line, `./backport-batch check --line --cut` has to exit 0. A cut tags the line, so the check fails while a pull request the rules take is not on the line, including one that is only on the open batch pull request: merge the batch pull request first. It prints the URLs, and prints what the batch deferred as warnings, since deferring is a maintainer's call. Without `--cut`, as the scheduled run uses it, a change on the open batch pull request counts as carried. +Before a patch release or an rc is cut from a line, `./backport-batch check --line --cut` has to exit 0. A cut tags the line, so the check fails while a pull request the rules take is not on the line, including one that is only on the open batch pull request: merge the batch pull request first. It prints the URLs, and prints what the batch deferred as warnings, since deferring is a maintainer's call; so is a pull request labelled `backport/skip` that the open batch still carries. An original that waits on the batch for a maintainer fails the cut, also when it is all the batch has left, and so does a unit on the batch that carries only part of its original, whatever its label. A cut that passes says so. Without `--cut`, as the scheduled run uses it, a change on the open batch pull request counts as carried. + +### The line's record and reports + +What the batch decided about a line, the conflicts it deferred, the originals taken out with `--defer` and those waiting for a maintainer, is kept in a `backport-batch` block in the body of the line's newest batch pull request, its record. `backport-batch` reads the record whether that pull request is open, merged or closed, and writes it only while the pull request is open: a merged or closed batch pull request is never edited or commented on again. So a conflict the batch deferred is still a warning in `check` after the batch merged, until a later batch tries it again. A run that picks nothing while no batch is open writes nothing, and says so. A record `backport-batch` cannot read, or one written by someone it does not trust, stops it rather than let it fall back to an older one; how it decides is in [its README](../../cmd/backport-batch/README.md#the-lines-record-and-reports). + +Each run also reports where every pull request merged into `main` since the line branched stands, and what it read to decide. The body carries the latest report under the record, and a comment the tool posts on the open batch pull request keeps each one, so an earlier classification is still there after a later run replaced the body, and after the batch merged. Each drop in a report keeps its reason and the merge commit it is bound to. `./backport-batch report --pr ` prints the report the body carries, and `--id ` an earlier one; it writes nothing and never fetches, and reads GitHub and, for a report that keeps drops, the clone's history at the heads the report was made from. A run that picks nothing while no batch is open has nowhere to keep its report: the next batch's report has the line's state. ### Changes to workflows -The scheduled run's token may not push a commit that changes `.github/workflows/`, so the scheduled run leaves such an original out (`run --no-workflow-files`). The batch pull request lists it under "Waiting for a maintainer", with the workflow files it changes, and so does the batch's comment. It is no conflict. The policy still takes it: `check` warns about it while it waits, and `check --cut` fails on it. When nothing else is picked there is no batch pull request to list it on, and `check` reports it as missing. +The scheduled run's token may not push a commit that changes `.github/workflows/`, so the scheduled run leaves such an original out (`run --no-workflow-files`). That holds also for one whose only missing dependency that never comes is a test or documentation, which a maintainer's run, with the token that may push the workflow, parks instead: what lands of it then is a pull request of its own; `check` reads which of the two the open batch was assembled as from its record, and with no batch open takes the scheduled run's. The batch pull request lists it under "Waiting for a maintainer", with the workflow files it changes, and so does the batch's comment. It is no conflict. The policy still takes it: `check` warns about it while it waits, and `check --cut` fails on it. When nothing else is picked there is no batch pull request to list it on, and `check` reports it as missing. A maintainer brings it over with their own token, one that may push workflow changes (the Workflows permission, or the classic `workflow` scope), by continuing the open batch without the flag: @@ -101,7 +117,7 @@ That picks it onto the batch in the batch's shape, after what the batch carries, ### The scheduled run -[`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) on Monday and Thursday. For each line the batch runs for, newest first, it continues the open batch pull request or opens one, runs `check`, and then `backport-batch notify`, which reports what waits on the batch. A batch open for three days or more, so one the previous run already saw, gets a comment on its pull request saying so and a line in the run's summary. The comment is one per pull request, updated in place on every run. It also lists the deferred conflicts, which a maintainer resolves as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. A failed batch CI waits for a maintainer too, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the comment does not list it, and it shows on the batch pull request's checks. `hack/backport-batch-scheduled.sh --dry-run` does the same in the current worktree and changes nothing on GitHub. Its `check` sees the line without the batch the dry run just built, so it can exit 1 where the real run's check passes. +[`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) on Monday and Thursday. For each line the batch runs for, newest first, it continues the open batch pull request or opens one, runs `check`, and then `backport-batch notify`, which reports what waits on the batch. A batch open for three days or more, so one the previous run already saw, gets a comment on its pull request saying so and a line in the run's summary. The comment is one per pull request, updated in place on every run. It also lists the deferred conflicts, which a maintainer resolves as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows, and names each parked original once: a park named on one batch is not named again on the next while it stays the same park, and with no batch open nothing is named until the next batch opens. A failed batch CI waits for a maintainer too, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the comment does not list it, and it shows on the batch pull request's checks. `hack/backport-batch-scheduled.sh --dry-run` does the same in the current worktree and changes nothing on GitHub. Its `check` sees the line without the batch the dry run just built, so it can exit 1 where the real run's check passes. The batch runs as the cozystack-ci GitHub App, which the release workflows already use, and needs no secret of its own. The job mints a token from `COZYSTACK_CI_APP_ID` and `COZYSTACK_CI_PRIVATE_KEY` for this repository alone, with Contents and Pull requests write and nothing more, and fails when either secret is missing. Its pushes start CI on the batch pull request, which a push with the job's own `GITHUB_TOKEN` would not. Its commits carry the App's bot, `cozystack-ci[bot]`, whose batch pull request and comment `backport-batch` trusts: `BACKPORT_BATCH_BOT`, which the job sets from the token, names it, and a run without the variable trusts `cozystack-ci[bot]` too. The App's installation has to grant this repository Contents and Pull requests read and write, which an organization admin sees among the organization's installed GitHub Apps. It has no Workflows permission, so a change to `.github/workflows/` waits for a maintainer, as [changes to workflows](#changes-to-workflows) describes. diff --git a/examples/backups/kafka/92-scenario-user-restore.md b/examples/backups/kafka/92-scenario-user-restore.md index 3215fe056..d637a6c9f 100644 --- a/examples/backups/kafka/92-scenario-user-restore.md +++ b/examples/backups/kafka/92-scenario-user-restore.md @@ -71,9 +71,6 @@ The script: had already replayed. The Pod log names each topic before its replay and again once it landed; empty or delete every topic it shows as started, not only the one the refusal names, then re-run. -- The tarball in object storage is not managed by the Cozystack `Backup` - lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 - object in place. Tenants who need to reclaim space should delete the object - (or set a bucket lifecycle policy). +- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. A Simple or MultiZone Cozystack SeaweedFS applies such rules once a day only while its `lifecycle.enabled` is on, which is not the default; a Client instance applies none itself, so the SeaweedFS it connects to must have `lifecycle.enabled` on. Another object store applies them only if it implements bucket lifecycle. - Consumer-group offsets, topic configs, ACLs and users are not part of the backup; consumers start from the restored topic's fresh offsets. diff --git a/examples/backups/kubernetes/00-helpers.sh b/examples/backups/kubernetes/00-helpers.sh new file mode 100755 index 000000000..401b5edb8 --- /dev/null +++ b/examples/backups/kubernetes/00-helpers.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Shared helpers for the tenant Kubernetes backup/restore walkthrough. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export NC='\033[0m' +export BOLD='\033[1m' + +export NAMESPACE="${NAMESPACE:-tenant-root}" +export CLUSTER_NAME="${CLUSTER_NAME:-backup-demo}" +export POOL_NAME="${POOL_NAME:-md0}" +export WORKER_DISK_SIZE="${WORKER_DISK_SIZE:-20Gi}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kubernetes-backup-job}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kubernetes-restore-inplace}" +export RESTOREJOB_RECREATE_NAME="${RESTOREJOB_RECREATE_NAME:-kubernetes-restore-recreate}" +export TENANT_API_PORT="${TENANT_API_PORT:-16443}" +export MARKER_VALUE="${MARKER_VALUE:-roundtrip-7c2e91d4}" +export RELEASE="kubernetes-${CLUSTER_NAME}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# The admin kubeconfig points at the in-cluster Service, so the walkthrough +# reaches the tenant API through a port-forward to it and keeps the TLS server +# name the certificate is issued for. +TENANT_KUBECONFIG="$(dirname "${BASH_SOURCE[0]}")/.tenant-kubeconfig" +export TENANT_KUBECONFIG + +tenant_connect() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-admin-kubeconfig" \ + -o jsonpath='{.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG" + chmod 600 "$TENANT_KUBECONFIG" + local cluster + cluster=$(kubectl --kubeconfig "$TENANT_KUBECONFIG" config view -o jsonpath='{.clusters[0].name}') + kubectl --kubeconfig "$TENANT_KUBECONFIG" config set-cluster "$cluster" \ + --server="https://127.0.0.1:${TENANT_API_PORT}" \ + --tls-server-name="${RELEASE}.${NAMESPACE}.svc" >/dev/null + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true + kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}" "${TENANT_API_PORT}:6443" >/dev/null 2>&1 & + for _ in $(seq 1 30); do + tkubectl get --raw /readyz >/dev/null 2>&1 && return 0 + sleep 2 + done + log_error "tenant API of ${RELEASE} is not reachable through the port-forward" + return 1 +} + +tenant_disconnect() { + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true +} + +tkubectl() { + kubectl --kubeconfig "$TENANT_KUBECONFIG" "$@" +} + +# The CA fingerprint is what proves a restore brought back the cluster rather +# than a new one: nodes, kubeconfigs and tokens all trust this CA. +cluster_ca_fingerprint() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' \ + | base64 -d | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2 +} + +# wait_tenant_marker checks both halves of the backup: the ConfigMap comes back +# with etcd, the file on the PVC with the volumes. +wait_tenant_marker() { + local timeout="${1:-900}" elapsed=0 cm data + while true; do + cm=$(tkubectl -n demo get configmap marker -o jsonpath='{.data.marker}' 2>/dev/null || true) + data=$(tkubectl -n demo exec deploy/writer -- cat /data/marker 2>/dev/null || true) + if [[ "$cm" == "$MARKER_VALUE" && "$data" == "$MARKER_VALUE" ]]; then + log_success "marker present in etcd (ConfigMap) and on the volume" + return 0 + fi + (( elapsed >= timeout )) && { log_error "marker not restored: configmap='${cm}' volume='${data}'"; return 1; } + sleep 10 + elapsed=$((elapsed + 10)) + done +} diff --git a/examples/backups/kubernetes/01-create-cluster.sh b/examples/backups/kubernetes/01-create-cluster.sh new file mode 100755 index 000000000..8577ec202 --- /dev/null +++ b/examples/backups/kubernetes/01-create-cluster.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Step 01: Create a tenant Kubernetes cluster with one worker pool and seed it +# with a marker held in etcd (a ConfigMap) and on a PersistentVolume. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 01: Create Kubernetes '${CLUSTER_NAME}' and seed a marker" + +kubectl apply -f - </dev/null 2>&1 \ + && [[ -n "$(tkubectl get nodes -o name)" ]] \ + && [[ -n "$(tkubectl get storageclass -o name)" ]]; then + kubectl_ok=1 + break + fi + sleep 15 +done +(( kubectl_ok )) || { log_error "worker node or StorageClass did not appear"; exit 1; } + +tkubectl apply -f - < /data/marker; sync; sleep infinity"] + volumeMounts: + - {name: data, mountPath: /data} + volumes: + - name: data + persistentVolumeClaim: {claimName: data} +EOF + +wait_tenant_marker 900 +cluster_ca_fingerprint > "$SCRIPT_DIR/.ca-fingerprint" +log_success "cluster CA $(cat "$SCRIPT_DIR/.ca-fingerprint")" +tenant_disconnect + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh" diff --git a/examples/backups/kubernetes/02-create-backupjob.sh b/examples/backups/kubernetes/02-create-backupjob.sh new file mode 100755 index 000000000..c290711c1 --- /dev/null +++ b/examples/backups/kubernetes/02-create-backupjob.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Step 02: Back the cluster up through the cozy-default BackupClass. The +# controller dumps the cluster's etcd prefix to the platform bucket with a +# Job, then backs up its control-plane Secrets and volumes with Velero. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'" + +kubectl apply -f - </dev/null || true) +kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found +[[ -n "$backup_ref" ]] && kubectl -n "$NAMESPACE" delete backup "$backup_ref" --ignore-not-found +kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found +kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --ignore-not-found +rm -f "$TENANT_KUBECONFIG" "$SCRIPT_DIR/.ca-fingerprint" +log_success "done" diff --git a/examples/backups/kubernetes/run-all.sh b/examples/backups/kubernetes/run-all.sh new file mode 100755 index 000000000..731448b4d --- /dev/null +++ b/examples/backups/kubernetes/run-all.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# End-to-end tenant Kubernetes backup/restore walkthrough. It proves the +# restore brings back the same cluster: the CA fingerprint is unchanged, and a +# marker kept in etcd and one kept on a PersistentVolume both come back, first +# after rolling a live cluster back, then after rebuilding a deleted one. +# +# Needs the platform default backups (backupstrategy-controller with a Velero +# BSL and the system bucket), so cozy-default routes Kubernetes to +# cozy-default-kubernetes. +# +# Env knobs: see 00-helpers.sh. SKIP_RESTORE=1 stops after the backup. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +kubectl wait --for=create kubernetes.strategy.backups.cozystack.io/cozy-default-kubernetes --timeout=15m >/dev/null \ + || { log_error "cozy-default-kubernetes strategy is missing: enable the platform default backups first"; exit 1; } + +steps=(01 02 03 04) +[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02) +for step in "${steps[@]}"; do + script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)" + [[ -n "$script" ]] || { echo "no script found for step $step" >&2; exit 1; } + echo "=== Running $(basename "$script") ===" + bash "$script" +done + +print_header "Walkthrough complete" diff --git a/examples/backups/nats/92-scenario-user-restore.md b/examples/backups/nats/92-scenario-user-restore.md index 9f0e7c02a..5aefface0 100644 --- a/examples/backups/nats/92-scenario-user-restore.md +++ b/examples/backups/nats/92-scenario-user-restore.md @@ -55,9 +55,6 @@ The script: - Both flows rely on the strategy template branching on `.Mode`. See `01-create-strategy.sh`. -- The tarball in object storage is not managed by the Cozystack `Backup` - lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 - object in place. Tenants who need to reclaim space should delete the object - (or set a bucket lifecycle policy). +- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. A Simple or MultiZone Cozystack SeaweedFS applies such rules once a day only while its `lifecycle.enabled` is on, which is not the default; a Client instance applies none itself, so the SeaweedFS it connects to must have `lifecycle.enabled` on. Another object store applies them only if it implements bucket lifecycle. - `nats stream restore` recreates the stream's configuration from the snapshot; consumers are restored as captured at backup time. diff --git a/go.mod b/go.mod index 34352e583..cb49f9f24 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/google/cel-go v0.29.2 github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 @@ -83,7 +84,6 @@ require ( github.com/go-openapi/testify/enable/yaml/v2 v2.8.0 // indirect github.com/go-openapi/testify/v2 v2.8.0 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe // indirect diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 0861da42d..070b6bad5 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -1,14 +1,28 @@ #!/usr/bin/env bats -# Behavioural tests for the shipped Altinity (ClickHouse) backup/restore -# strategy script, extracted from the rendered cozy-default-altinity Strategy -# CR and EXECUTED against a stubbed clickhouse-backup HTTP API, with the real -# jq doing the archive selection. Which archive a restore reads back is -# decided by that selection over list/remote, so it is exercised against -# several archives rather than string-matched. +# Behavioural tests for the shipped ClickHouse (Altinity) strategy script, +# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED +# against a stubbed clickhouse-backup sidecar API. # -# The script runs with `bash`, not `sh`, for the reason given in -# hack/redis-backup-strategy.bats: it opens `set -euo pipefail`, which the -# production image's busybox ash supports and the CI runner's dash rejects. +# The contract pinned here is the start gate: the sidecar serialises its +# operations and answers any new one with HTTP 423 while another is "in +# progress", so a strategy Pod that starts its operation without waiting for +# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod +# rather than proceed when the sidecar never becomes idle. Once started, an +# operation the sidecar no longer lists (its action log lives in memory and a +# restart empties it) has to fail the Pod too, since the Job has no deadline. +# A Job retry must not start its operation twice: it takes over the +# restore_remote, or the create_remote of the archive the controller named, +# that the attempt before it left running, and a few lost polls do not end an +# attempt in the first place. +# +# Which archive a restore reads back is decided by the script's own jq +# selection over list/remote, so it is exercised against several archives +# rather than string-matched: exactly the archive the Backup recorded, or for +# a Backup that records none, the newest one stamped no later than its takenAt. +# +# Run from the repo root with make bats-unit-tests. The rendered script is run +# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production +# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects. load test_helper @@ -16,12 +30,13 @@ CHART=packages/system/backupstrategy-controller # Render the shipped strategy's in-Pod script into $1 and substitute the # controller-render placeholders the way the driver fills them: $2 mode, $3 -# app (release and source application), $4 the archive name recorded on the -# Backup (empty for a Backup that records none), $5 the Backup name, $6 its -# takenAt in the archive-name layout. An empty render fails early, as does a -# placeholder left unsubstituted. +# app (release and source application), $4 the archive name the controller +# chose or the Backup recorded (empty, the default, for none), $5 the Backup +# name, $6 its takenAt in the archive-name layout (default: later than every +# archive the stub lists). An empty render fails early, as does a placeholder +# left unsubstituted: these tests EXECUTE it. render_script() { - out=$1; mode=$2; app=$3; archive=$4; backup=$5; takenat=$6 + out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z} helm template t "$CHART" -n cozy-backup-controller \ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ @@ -39,56 +54,118 @@ render_script() { fi } -# Stub the clickhouse-backup API behind curl, first on PATH. list/remote -# serves $state/remote; actions serves $state/actions plus one success row per -# operation the script has issued, and after its first read swaps in -# $state/actions.next when a test seeds one (an operation that finishes while -# the script waits). Every request is logged to $state/curl.log. sleep is -# stubbed to fail after a few calls, so a script that never sees its -# operation finish fails the test instead of hanging it. +# A curl stub standing in for the sidecar API, put first on PATH next to a +# no-op sleep. State lives in $1: +# unreachable present: every request fails as a refused connection (7) +# down a number: that many next requests fail the same way, as +# before the sidecar has started +# busy a number: that many GET /backup/actions answers still list +# an operation of someone else "in progress" +# op the command line of the sidecar's own operation, written +# by an accepted POST the way the sidecar logs it ("--resume" +# before the name of a create_remote asking for it), or +# seeded by a test as the operation +# an earlier attempt of the same Job started +# op_left how many more GET /backup/actions answers list op as +# "in progress" before it ends with op_result +# op_duration op_left given to an operation a POST starts (default 0) +# op_result the status op ends with (default success) +# prior extra action-log rows, verbatim, older than op +# remote present: the list/remote answer, verbatim, in place of the +# one archive listed by default +# restarted present: the action log no longer lists op, as after the +# sidecar restarted and lost its in-memory log +# fail_on_post a POST that is accepted makes that many following +# GET /backup/actions requests fail (curl exit 7) +# idle created by an answer that lists nothing in progress; a +# POST is refused (423, curl -f exit 22) unless one was seen +# since the last accepted POST, and while anything runs +# accepted one line per POST the sidecar accepted +# requests one line per request: method and URL +# runaway created when the script exceeds 200 requests; that request +# and every later one fails, so a polling loop that never ends +# stops the test instead of hanging it make_stubs() { bin=$1; state=$2 - mkdir -p "$bin" - : > "$state/remote" - : > "$state/actions" + mkdir -p "$bin" "$state" cat > "$bin/curl" <> "$state/curl.log" +echo "\$method \$url" >> "\$state/requests" +if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi +if [ -e "\$state/unreachable" ]; then exit 7; fi +num() { cat "\$state/\$1" 2>/dev/null || echo 0; } +down=\$(num down) +if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi +running=0 +if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi case "\$method \$url" in - "GET "*/backup/status) echo '{}' ;; - "GET "*/backup/list/remote) cat "$state/remote" ;; + "POST "*) + if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi + case "\$url" in + */create_remote\?name=*) + name=\${url#*name=}; name=\${name%%&*} + case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac + ;; + */restore_remote/*) cmd="restore_remote \${url##*/}" ;; + *) exit 22 ;; + esac + echo "\$cmd" > "\$state/op" + echo "\$cmd" >> "\$state/accepted" + num op_duration > "\$state/op_left" + num fail_on_post > "\$state/fail_polls" + rm -f "\$state/idle" + ;; "GET "*/backup/actions) - cat "$state/actions" - if [ -f "$state/actions.done" ]; then cat "$state/actions.done"; fi - if [ -f "$state/actions.next" ]; then mv "$state/actions.next" "$state/actions"; fi + fail=\$(num fail_polls) + if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi + echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}' + cat "\$state/prior" 2>/dev/null || true + inprog=0 + busy=\$(num busy) + if [ "\$busy" -gt 0 ]; then + echo \$((busy - 1)) > "\$state/busy" + echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}' + inprog=1 + fi + if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then + left=\$(num op_left) + if [ "\$left" -gt 0 ]; then + echo \$((left - 1)) > "\$state/op_left" + status="in progress" + inprog=1 + else + status=\$(cat "\$state/op_result" 2>/dev/null || echo success) + fi + echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}" + fi + if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi ;; - "POST "*/backup/create_remote*) - name=\${url#*name=}; name=\${name%%&*} - printf '{"command":"create_remote --resume %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\$name" >> "$state/actions.done" + "GET "*/backup/list/remote) + if [ -e "\$state/remote" ]; then cat "\$state/remote"; else + echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + fi ;; - "POST "*/backup/restore_remote/*) - printf '{"command":"restore_remote %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\${url##*/}" >> "$state/actions.done" - ;; - *) echo "unexpected request: \$method \$url" >&2; exit 22 ;; esac +exit 0 STUB - cat > "$bin/sleep" </dev/null || echo 0) -echo \$((n + 1)) > "$state/sleeps" -if [ "\$n" -ge 3 ]; then echo "sleep budget spent: the operation never finished" >&2; exit 1; fi + cat > "$bin/sleep" <<'STUB' +#!/bin/sh +exit 0 STUB chmod 0755 "$bin/curl" "$bin/sleep" - export API_USERNAME=u API_PASSWORD=p } # One list/remote row per argument, "name" or "name|desc"; created mirrors -# the name's timestamp, which is the order the pre-fix selection sorted by. +# the name's timestamp. remote_rows() { for row in "$@"; do name=${row%%|*}; desc="tar, regular" @@ -98,49 +175,216 @@ remote_rows() { done } +accepted() { + if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi +} + +run_script() { + tmp=$1 + API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" +} + @test "an empty render fails the fixture instead of passing as a no-op" { tmp=$(mktemp -d) CHART=$tmp/no-such-chart - if render_script "$tmp/s.sh" restore myapp "" bkp 20260304T050622Z 2>/dev/null; then + if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then echo "FAIL: render_script reported success with no chart" false fi - rm -rf "$tmp" +} + +@test "backup on an idle sidecar starts create_remote and waits for its success" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" +} + +@test "backup waits until the running operation finishes before create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted" + [ "$(cat "$tmp/state/busy")" = 0 ] +} + +@test "restore on an idle sidecar starts restore_remote of the listed archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + grep -q '^POST .*/backup/restore_remote/clickhouse-myapp-20260101T000000Z$' "$tmp/state/requests" +} + +@test "backup fails without starting create_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi +} + +@test "backup fails without starting create_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi +} + +@test "restore fails without starting restore_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on an unreachable sidecar"; false; fi + grep -q 'list/remote failed 60 times in a row' "$tmp/err" + [ "$(grep -c '^GET .*/backup/list/remote$' "$tmp/state/requests")" = 60 ] +} + +@test "restore fails without starting restore_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi +} + +@test "backup fails instead of polling forever when the sidecar restarts after create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/restarted" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi + grep -q 'no longer in the sidecar action log' "$tmp/err" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi +} + +@test "restore waits for a sidecar that has not started yet" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 10 > "$tmp/state/down" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "restore waits until another operation finishes, then starts its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/op_duration" + echo 5 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a retry after an attempt lost its polls joins the running restore instead of starting another" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 20 > "$tmp/state/op_duration" + echo 6 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi + [ "$(accepted "$tmp")" = 1 ] + echo 0 > "$tmp/state/fail_polls" + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "restore does not join an operation running on another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +# The sidecar never trims its action log, so a finished restore_remote row of +# the same archive may belong to an earlier RestoreJob; only a running one may +# be taken over. +@test "restore starts restore_remote again when the log holds a finished one of the same archive" { + for status in success error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi + done +} + +@test "a restore that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a cancelled backup fails the Pod instead of polling forever" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo cancel > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi } @test "restore reads back the archive the Backup recorded, not the newest one" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ - clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] - rm -rf "$tmp" + clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] } @test "restore fails naming the recorded archive when remote storage lacks it" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z - remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded without the recorded archive: $out" false fi printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored another archive: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored another archive: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z # Each decoy sorts after the expected archive and is ruled out by one # filter: broken, not a bare timestamp after the prefix, or stamped after @@ -148,71 +392,140 @@ remote_rows() { remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ clickhouse-myapp-20260304T050710Z-copy \ - clickhouse-myapp-20260304T060000Z > "$tmp/remote" - out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1) - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] + myapp-20260304050720-0f3c9a1e5b7d \ + clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" + out=$(run_script "$tmp" 2>&1) + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z - remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded with every archive newer than takenAt: $out" false fi printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "backup writes the archive under the name the controller chose" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" -} - -@test "a retried backup waits on the attempt the sidecar is still running" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"in progress","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions.next" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - # The sidecar answers a second operation with 423 while one runs. - if grep -qF 'POST ' "$tmp/curl.log"; then - echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")" - false - fi - rm -rf "$tmp" -} - -@test "a retried backup after a failed attempt resumes the same archive" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" } @test "backup falls back to a timestamped name when the controller supplies none" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp "" "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp "" + run_script "$tmp" + grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests" +} + +# The sidecar answers a second operation with 423 while one runs, so a retry +# that POSTed over its predecessor's upload would burn every attempt left. +@test "a retried backup takes over the upload of its archive the sidecar is still running" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "a taken-over backup that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 2 > "$tmp/state/op_left" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup after a failed attempt resumes the same archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" +} + +@test "backup does not take over an upload running for another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted" +} + +# The archive is named per BackupJob, so a success row of exactly this +# upload is an earlier attempt of the same BackupJob that finished while it +# could not see it. Uploading again with resume would re-upload over the +# finished archive and break it if that second upload dies halfway. +@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" { + for status in error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi + done +} + +@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + printf '%s\n' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "backup does not take another archive's finished upload as its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] } diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats new file mode 100644 index 000000000..e912be42f --- /dev/null +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -0,0 +1,263 @@ +#!/usr/bin/env bats +# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile +# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a +# restore that fails without saying so leaves the rewrite in a cluster-wide +# object and still reports a green run; the next run then refuses to start. +# These run the restore against a stubbed kubectl. + +load test_helper + +LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh +HELPERS=examples/backups/clickhouse/00-helpers.sh + +setup() { + strict_setup + STUB=$(mktemp -d) + # A Corefile carrying the Test's marked block, as the run step leaves it. + jq -n '{apiVersion: "v1", kind: "ConfigMap", + metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"}, + data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \ + >"$STUB/coredns.json" + # The secret lookup follows kubectl's contract: a name together with a + # selector is refused, and a selector-only lookup prints the names in + # OWNED_SECRETS, the Secrets carrying the owner label. + cat >"$STUB/kubectl" <<'EOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;; + replace*) + cp "${@: -1}" "$STUB/replaced.json" + if [ -n "${REPLACE_FAILS:-}" ]; then + echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2 + exit 1 + fi + ;; + *"get secret"*) + # Positionals are "get", "secret" and then any names. + words=0 selector= + while [ $# -gt 0 ]; do + case "$1" in + -l|--selector) selector=$2; shift 2 ;; + -n|--namespace|-o|--output) shift 2 ;; + -*) shift ;; + *) words=$((words + 1)); shift ;; + esac + done + if [ -n "${LOOKUP_FAILS:-}" ]; then + echo 'Unable to connect to the server: dial tcp: i/o timeout' >&2 + exit 1 + fi + if [ -n "$selector" ] && [ "$words" -gt 2 ]; then + echo 'error: name cannot be provided when a selector is specified' >&2 + exit 1 + fi + [ "$selector" = cozystack.io/e2e-owner=clickhouse-system-bucket ] || exit 0 + for s in ${OWNED_SECRETS:-}; do echo "secret/$s"; done + ;; + *"delete secret "*) + if [ -n "${DELETE_FAILS:-}" ]; then + echo 'Error from server (Forbidden): secrets "seaweedfs-s3-ingress-tls" is forbidden' >&2 + exit 1 + fi + ;; + *) echo "unexpected kubectl $*" >&2; exit 2 ;; +esac +EOF + chmod +x "$STUB/kubectl" + export STUB +} + +restore() { + PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c ' + set -euo pipefail + source "$1" + source "$2" + set +e + restore_shared_state + ' _ "$HELPERS" "$LIB" +} + +@test "a CoreDNS restore the API server refuses fails the restore" { + export REPLACE_FAILS=1 + err=$(mktemp) + if restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl replace was refused" >&2 + cat "$err" >&2 + false + fi + if grep -q 'Removed the s3.example.org rewrite' "$err"; then + echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2 + cat "$err" >&2 + false + fi +} + +@test "a CoreDNS restore sends the Corefile back without the marked block" { + restore + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || { + echo "restored Corefile:" >&2 + jq -r '.data.Corefile' "$STUB/replaced.json" >&2 + false + } + # The resourceVersion read travels with the replace, so a concurrent + # writer makes it fail rather than be overwritten. + [ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ] +} + +@test "the restore deletes the Ingress TLS Secret the Test created" { + OWNED_SECRETS=seaweedfs-s3-ingress-tls restore + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state did not delete the Secret carrying the owner label; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + } +} + +@test "the restore leaves an Ingress TLS Secret without the owner label alone" { + for owned in '' e2e-other seaweedfs-s3-ingress-tls-old; do + : >"$STUB/calls" + OWNED_SECRETS=$owned restore + if grep -q 'delete secret' "$STUB/calls"; then + echo "restore_shared_state deleted a Secret although the labelled ones are '$owned'; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + fi + done +} + +@test "a Secret delete the API server refuses fails the restore" { + err=$(mktemp) + if OWNED_SECRETS=seaweedfs-s3-ingress-tls DELETE_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl delete secret was refused" >&2 + cat "$err" >&2 + false + fi + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state failed before it reached the Secret delete; kubectl calls:" >&2 + cat "$STUB/calls" "$err" >&2 + false + } +} + +@test "a Secret lookup that fails fails the restore" { + err=$(mktemp) + if LOOKUP_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although it could not tell whether the Secret is left over" >&2 + cat "$err" >&2 + false + fi + grep -q 'could not list the Secrets labelled' "$err" || { + echo "restore_shared_state failed without naming the Secret lookup:" >&2 + cat "$err" >&2 + false + } +} + +@test "the run step's finally restores the shared objects" { + # When the step ceiling fires, Chainsaw kills the shell and the EXIT trap + # of system-bucket-roundtrip.sh never runs; the step's finally still does. + # Chainsaw runs a script's content with sh from the suite's directory. + suite=hack/e2e-chainsaw/clickhouse + finally=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") + | .spec.steps[] | select(.name == "run") | .finally[] + | select(.description == "restore the host CoreDNS and the S3 Ingress TLS Secret") | .script.content' \ + "$suite/chainsaw-test.yaml") + [ -n "$finally" ] || { echo "the run step has no finally restoring the shared objects" >&2; false; } + (cd "$suite" && PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$finally") + [ -f "$STUB/replaced.json" ] || { + echo "the run step's finally did not put the Corefile back" >&2 + false + } + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null +} + +# The Altinity driver names the Backup after the BackupJob, and a Backup left +# behind under that name records an earlier run's archive, which fails the next +# run's BackupJob. It is reclaimed by name, whether or not the BackupJob +# recorded it in status.backupRef. +@test "the round-trip reclaims its Backup by name before and after the run" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + # Nothing is left over from an earlier run, and no BackupJob recorded a + # backupRef. + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + for selector in \ + 'select(.name == "preconditions") | .try[].script.content' \ + 'select(.name == "run") | .finally[] | select(.description == "system-bucket round-trip cleanup") | .script.content'; do + : >"$calls" + script=$(yq "$test | $selector" "$suite") + [ -n "$script" ] || { echo "no script at: $selector" >&2; false; } + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + grep -qE '^-n tenant-root delete backup sysbucket-backup( |$)' "$calls" || { + echo "the script at '$selector' does not delete Backup sysbucket-backup by name; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + done +} + +# cert-manager issues the Secret of a Certificate that still exists again as +# soon as it is deleted. Chainsaw deletes the Certificate it applied, and waits +# for it, before it runs the cleanup ops of the same step, so the Secret is +# deleted there and nowhere earlier. +@test "the Certificate's Secret is deleted after Chainsaw deletes the Certificate" { + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + applies=$(yq "$test | select(.name == \"create-certificate-and-probe\") | .try[] | select(has(\"apply\")) | .apply.file" "$suite") + [ "$applies" = system-bucket-tls.yaml ] || { + echo "step create-certificate-and-probe no longer applies system-bucket-tls.yaml alone: $applies" >&2 + false + } + deletes=$(yq -o=json -I=0 "$test | select(.name == \"create-certificate-and-probe\") | .cleanup" "$suite") + [ "$deletes" = '[{"delete":{"ref":{"apiVersion":"v1","kind":"Secret","name":"e2e-system-bucket-s3-tls"}}}]' ] || { + echo "the cleanup of step create-certificate-and-probe is not the one delete of Secret e2e-system-bucket-s3-tls: $deletes" >&2 + false + } + # The preconditions run before the apply, when Chainsaw holds no Certificate. + if yq "$test | select(.name != \"preconditions\") | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then + echo "a step script deletes the Certificate or its Secret while Chainsaw still holds the Certificate" >&2 + false + fi +} + +# Chainsaw registers an applied object for deletion only when its apply created +# it; an object left by a run killed past its cleanup is patched and kept. The +# Certificate would then outlive every later run, re-issuing its Secret after +# each step cleanup, and a probe Pod whose `sleep` had ended never turns Ready +# again, so the preconditions remove all three before the apply. +@test "the preconditions remove the TLS objects and probe Pod a killed run left" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + : >"$calls" + script=$(yq "$test | select(.name == \"preconditions\") | .try[].script.content" "$suite") + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + line_of() { grep -nE "^-n tenant-root delete $1 --ignore-not-found --wait=true( |$)" "$calls" | head -n1 | cut -d: -f1; } + cert=$(line_of 'certificate e2e-system-bucket-s3') + secret=$(line_of 'secret e2e-system-bucket-s3-tls') + pod=$(line_of 'pod e2e-system-bucket-probe') + [ -n "$cert" ] && [ -n "$secret" ] && [ -n "$pod" ] || { + echo "the preconditions do not delete, and wait for, the Certificate, its Secret and the probe Pod; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + [ "$cert" -lt "$secret" ] || { + echo "the preconditions delete Secret e2e-system-bucket-s3-tls before its Certificate, which cert-manager then issues again" >&2 + false + } +} diff --git a/hack/console-nginx-header-buffers.bats b/hack/console-nginx-header-buffers.bats index f8ce4d949..21932fc25 100644 --- a/hack/console-nginx-header-buffers.bats +++ b/hack/console-nginx-header-buffers.bats @@ -37,11 +37,11 @@ nginx_size_kib() { cd "$REPO_ROOT" conf=$(awk ' - /^COPY <<.EOF. \/etc\/nginx\/conf\.d\/default\.conf$/ { inblock = 1; next } + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } inblock && /^EOF$/ { inblock = 0; next } inblock { print } ' "$CONSOLE_CONTAINERFILE") - [ -n "$conf" ] || { echo "no /etc/nginx/conf.d/default.conf heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; } + [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; } # An extraction that silently grabbed the wrong slice would let every # assertion below pass on an empty-ish config, so anchor on a line the diff --git a/hack/console-nginx-observability-proxy.bats b/hack/console-nginx-observability-proxy.bats new file mode 100644 index 000000000..de2348dd7 --- /dev/null +++ b/hack/console-nginx-observability-proxy.bats @@ -0,0 +1,243 @@ +#!/usr/bin/env bats +# The console nginx sends the logs and metrics roots of the observability +# proxy to that proxy instead of the API server. The routing lives in the +# nginx config baked into the console image, which no chart template and no +# helm-unittest case can see, so it is pinned here. +# +# Four properties are checked. The location regex must take the four path +# shapes the proxy serves and leave every ordinary Kubernetes API path to the +# /apis and /api prefix locations; nginx runs the regex locations whenever the +# longest prefix match is a plain prefix, so the prefix locations must not +# carry ^~, which would stop that. The location must stream: tail and watch +# style responses stall behind proxy buffering and die at the default 60s +# read timeout. And the upstream must be a variable resolved per request, +# because nginx resolves a literal upstream host once at startup and refuses +# to start when it does not resolve, which is the case on every cluster that +# does not run the proxy. Finally, when nginx cannot reach the proxy it +# answers with the Kubernetes Status the console reads as not installed, +# while the proxy's own 502 passes through; that check runs the extracted +# locations in a real nginx and is skipped where none is installed. +# +# The regex is evaluated with perl, whose syntax nginx's PCRE accepts for the +# constructs used here. +# Run with: bats hack/console-nginx-observability-proxy.bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile" + +console_nginx_template() { + awk ' + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } + inblock && /^EOF$/ { inblock = 0; next } + inblock { print } + ' "$REPO_ROOT/$CONSOLE_CONTAINERFILE" +} + +# Prints the regex of every `location ~ {` line, one per line. +regex_locations() { + printf '%s\n' "$1" | awk '$1 == "location" && $2 == "~" && $4 == "{" { print $3 }' +} + +# Prints the body of the location block whose regex is $2. +location_body() { + # Through ENVIRON rather than -v, which would expand the backslashes. + printf '%s\n' "$1" | LOCATION_RE="$2" awk ' + $1 == "location" && $2 == "~" && $3 == ENVIRON["LOCATION_RE"] { inblock = 1; next } + inblock && $1 == "}" { exit } + inblock { print } + ' +} + +matches() { + perl -e 'exit(($ARGV[1] =~ /$ARGV[0]/) ? 0 : 1)' "$1" "$2" +} + +@test "console nginx routes exactly the observability proxy roots to the proxy" { + conf=$(console_nginx_template) + [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc in $CONSOLE_CONTAINERFILE" >&2; exit 1; } + + regexes=$(regex_locations "$conf") + [ "$(printf '%s\n' "$regexes" | grep -c .)" -eq 1 ] \ + || { echo "expected exactly one regex location, got: $regexes" >&2; exit 1; } + re=$regexes + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/metrics/api/v1/query_range \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log/select/logsql/tail \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/vminstances/vm1/metrics/api/v1/query + do + matches "$re" "$path" || { echo "$path does not reach the proxy" >&2; exit 1; } + done + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/workloadmonitors/db \ + /apis/cozystack.io/v1alpha1/applicationdefinitions/postgres \ + /apis/cozystack.io/v1alpha1/watch/namespaces/tenant-foo/logs/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/status \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log \ + /apis/apps.cozystack.io/v1alpha1/watch/namespaces/tenant-foo/postgreses/db/log/ \ + /apis/metrics.k8s.io/v1beta1/namespaces/tenant-foo/pods/ \ + /api/v1/namespaces/tenant-foo/pods/db-0/log \ + /k8s/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query + do + if matches "$re" "$path"; then + echo "FAIL: $path would reach the proxy instead of the API server" >&2 + false + fi + done + + # A ^~ on the /apis or /api prefix would stop nginx from trying the regex. + if printf '%s\n' "$conf" | grep -Eq '^[[:space:]]*location[[:space:]]+\^~'; then + echo "FAIL: a ^~ prefix location disables the regex location" >&2 + false + fi +} + +@test "the observability proxy location streams, forwards the token and resolves per request" { + conf=$(console_nginx_template) + re=$(regex_locations "$conf") + body=$(location_body "$conf" "$re") + [ -n "$body" ] || { echo "no body found for location ~ $re" >&2; exit 1; } + + for directive in \ + 'proxy_http_version 1.1;' \ + 'proxy_buffering off;' \ + 'proxy_read_timeout 86400s;' \ + 'proxy_send_timeout 86400s;' + do + printf '%s\n' "$body" | grep -Fqx " $directive" \ + || { echo "the proxy location lacks '$directive'" >&2; exit 1; } + done + + # nginx forwards client headers unless told otherwise; the proxy needs the + # Authorization header the gatekeeper set. + if printf '%s\n' "$body" | grep -Eq 'proxy_set_header[[:space:]]+Authorization|proxy_pass_request_headers[[:space:]]+off'; then + echo "FAIL: the proxy location drops or rewrites the Authorization header" >&2 + false + fi + + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*resolver[[:space:]]+\$\{NGINX_LOCAL_RESOLVERS\}' \ + || { echo "the proxy location sets no resolver from the pod's resolv.conf" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*set[[:space:]]+\$observability_proxy[[:space:]]+observability-proxy\.cozy-observability-proxy\.svc\.\$\{CLUSTER_DOMAIN\}:80;' \ + || { echo "the proxy location does not name the Service by its fully qualified name" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*proxy_pass[[:space:]]+http://\$observability_proxy;' \ + || { echo "the proxy location does not proxy_pass through the variable" >&2; exit 1; } +} + +@test "the console image defines every variable its nginx template substitutes" { + cd "$REPO_ROOT" + conf=$(console_nginx_template) + + # The image's entrypoint substitutes only variables present in the + # environment; one that is not would reach nginx as ${NAME} and fail it. + grep -Eq '^ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image does not ask its entrypoint for NGINX_LOCAL_RESOLVERS" >&2; exit 1; } + grep -Eq '^ENV CLUSTER_DOMAIN=cozy\.local$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image has no CLUSTER_DOMAIN default" >&2; exit 1; } + + used=$(printf '%s\n' "$conf" | grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' | sort -u) + expected=$(printf '%s\n' '${CLUSTER_DOMAIN}' '${NGINX_LOCAL_RESOLVERS}') + [ "$used" = "$expected" ] \ + || { echo "template substitutes: $used; the image defines: $expected" >&2; exit 1; } +} + +# Prints the named location $2 of the template $1, braces included. +named_location() { + printf '%s\n' "$1" | LOCATION_NAME="$2" awk ' + $1 == "location" && $2 == ENVIRON["LOCATION_NAME"] { inblock = 1 } + inblock { print } + inblock && $1 == "}" { exit } + ' +} + +free_port() { + python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])' +} + +# Starts nginx with the observability locations of the console template, the +# upstream pointed at 127.0.0.1:$1, and a stand-in proxy on 127.0.0.1:$2 +# that answers 502 itself. Prints the console port. +start_console_nginx() { + upstream_port=$1 standin_port=$2 + conf=$(console_nginx_template) + re=$(regex_locations "$conf") + console_port=$(free_port) + { + echo "pid $BATS_TEST_TMPDIR/nginx.pid;" + echo "error_log $BATS_TEST_TMPDIR/error.log;" + echo "events {}" + echo "http {" + echo " access_log off;" + echo " client_body_temp_path $BATS_TEST_TMPDIR; proxy_temp_path $BATS_TEST_TMPDIR;" + echo " fastcgi_temp_path $BATS_TEST_TMPDIR; uwsgi_temp_path $BATS_TEST_TMPDIR; scgi_temp_path $BATS_TEST_TMPDIR;" + echo " server {" + echo " listen 127.0.0.1:$console_port;" + echo " location ~ $re {" + # The test upstream is an address, so the resolver is never asked. + location_body "$conf" "$re" \ + | sed -e 's/\${NGINX_LOCAL_RESOLVERS}/127.0.0.1/' \ + -e "s/^\([[:space:]]*set[[:space:]]*\$observability_proxy[[:space:]]\).*/\1127.0.0.1:$upstream_port;/" + echo " }" + named_location "$conf" "@observability_proxy_unavailable" + echo " }" + echo " server {" + echo " listen 127.0.0.1:$standin_port;" + echo " location / { return 502 'log store unreachable'; }" + echo " }" + echo "}" + } > "$BATS_TEST_TMPDIR/nginx.conf" + nginx -p "$BATS_TEST_TMPDIR" -c "$BATS_TEST_TMPDIR/nginx.conf" \ + || { cat "$BATS_TEST_TMPDIR/error.log" >&2; return 1; } + echo "$console_port" +} + +teardown() { + if [ -f "${BATS_TEST_TMPDIR:-}/nginx.pid" ]; then + kill "$(cat "$BATS_TEST_TMPDIR/nginx.pid")" 2>/dev/null || true + fi +} + +@test "a cluster without the proxy answers the console's not-installed Status, the proxy's own 502 passes through" { + command -v nginx >/dev/null 2>&1 || skip "nginx is not installed" + command -v curl >/dev/null 2>&1 || skip "curl is not installed" + path=/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query + + # Nothing listens on the upstream port: nginx itself fails the request. + standin=$(free_port) + port=$(start_console_nginx "$(free_port)" "$standin") + out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path") + printf '%s\n' "$out" | head -n 1 | grep -q ' 404' \ + || { echo "FAIL: an unreachable proxy did not answer 404: $out" >&2; false; } + printf '%s\n' "$out" | grep -iq '^content-type: application/json' \ + || { echo "FAIL: the not-installed answer is not JSON: $out" >&2; false; } + body=$(printf '%s\n' "$out" | tail -n 1) + printf '%s' "$body" | python3 -c ' +import json, sys +s = json.load(sys.stdin) +assert s["kind"] == "Status" and s["apiVersion"] == "v1" and s["status"] == "Failure", s +assert s["reason"] == "NotFound" and s["code"] == 404, s +assert s["message"] == "the observability proxy is not available on this cluster", s +' || { echo "FAIL: the not-installed body is not the expected Status: $body" >&2; false; } + # The next nginx gets fresh ports, so the old master need not have + # released its sockets; it is still stopped first to keep one running. + pid=$(cat "$BATS_TEST_TMPDIR/nginx.pid") + kill "$pid" + for _ in $(seq 1 50); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + rm -f "$BATS_TEST_TMPDIR/nginx.pid" + + # The proxy answers 502 itself: the console must see that answer. + standin=$(free_port) + port=$(start_console_nginx "$standin" "$standin") + out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path") + printf '%s\n' "$out" | head -n 1 | grep -q ' 502' \ + || { echo "FAIL: the proxy's own 502 did not pass through: $out" >&2; false; } + printf '%s\n' "$out" | tail -n 1 | grep -qx 'log store unreachable' \ + || { echo "FAIL: the proxy's own 502 body was replaced: $out" >&2; false; } +} diff --git a/hack/cozyreport.bats b/hack/cozyreport.bats index cc2668071..ee8911080 100644 --- a/hack/cozyreport.bats +++ b/hack/cozyreport.bats @@ -966,6 +966,37 @@ STUB } } +@test "PR E2E jobs clean up between preparation attempts but not after testing" { + tmp="$(mktemp -d)" + mkdir -p "$tmp/bin" + cat >"$tmp/bin/make" <<'STUB' +#!/bin/sh +printf '%s\n' "$*" >>"$CLEANUP_CALLS" +case "$*" in + *prepare-env-container*) + if [ ! -f "$CLEANUP_ATTEMPT" ]; then + touch "$CLEANUP_ATTEMPT" + exit 1 + fi + ;; +esac +STUB + chmod +x "$tmp/bin/make" + for wf in pull-requests e2e-fork; do + workflow="$HACK_DIR/../.github/workflows/$wf.yaml" + yq --unwrapScalar '.jobs.e2e.steps[] | select(.name == "Prepare environment") | .run' "$workflow" >"$tmp/prepare.sh" + CLEANUP_CALLS="$tmp/$wf.calls" CLEANUP_ATTEMPT="$tmp/$wf.attempt" SANDBOX_NAME=test PATH="$tmp/bin:$PATH" bash -eu "$tmp/prepare.sh" + printf '%s\n' 'SANDBOX_NAME=test prepare-env-container' '-C packages/core/testing SANDBOX_NAME=test delete-cluster-container' 'SANDBOX_NAME=test prepare-env-container' >"$tmp/expected" + diff "$tmp/expected" "$tmp/$wf.calls" + scripts="$(yq --unwrapScalar '.jobs.e2e.steps[] | select(.name != "Prepare environment") | .run // ""' "$workflow")" + if printf '%s\n' "$scripts" | grep -Eq 'delete-cluster-container|docker[[:space:]]+(compose.*down|rm)|zpool[[:space:]]+destroy|make.*[[:space:]]delete([[:space:]]|$)'; then + echo "FAIL: $wf destroys runner-local state outside preparation" + false + fi + done + rm -rf "$tmp" +} + @test "the step that runs this collector has a ceiling of its own everywhere" { # Every bound inside the script protects one read. None of them protects the # STEP: the tarball is written on the collector's last line, the upload is the diff --git a/hack/cozyreport.sh b/hack/cozyreport.sh index 0f20b1261..0e6c12b15 100755 --- a/hack/cozyreport.sh +++ b/hack/cozyreport.sh @@ -121,7 +121,7 @@ # the number; do not expect to find it written here. # # TWO ceilings sit outside it, and they fail differently. The job is capped at -# `timeout-minutes: 215`, and reaching that cancels every step still to come -- +# `timeout-minutes: 265`, and reaching that cancels every step still to come -- # the artifact upload among them. The step running this script is capped at 30 of # its own, and reaching THAT ends only this step: the uploads after it still run, # and `continue-on-error` on that step keeps the overrun out of the job's result, diff --git a/hack/cozytest.sh b/hack/cozytest.sh index 02a40f383..257bdcd08 100755 --- a/hack/cozytest.sh +++ b/hack/cozytest.sh @@ -208,7 +208,7 @@ _cozy_on_exit() { # capture would read as a complete one listing fewer crash-loops than # actually occurred. This handler's whole worst case is ~1350s (330 + 390 # + 630, each leg including its 30s kill grace) against a job capped at - # 215 minutes with no per-step timeout — + # 265 minutes with no per-step timeout — # deliberate, for the same reason recorded in hack/e2e-chainsaw/.chainsaw.yaml: # a bounded, honest wait beats a kill that collects nothing. if [ "$_prev_rc" -ne 0 ]; then diff --git a/hack/e2e-chainsaw/.chainsaw.yaml b/hack/e2e-chainsaw/.chainsaw.yaml index 87aa95619..6748a91c6 100644 --- a/hack/e2e-chainsaw/.chainsaw.yaml +++ b/hack/e2e-chainsaw/.chainsaw.yaml @@ -66,14 +66,14 @@ spec: # script: it SIGKILLs the shell alone, orphaned children hold the pipe open, # and the op overruns (#4268 measured 606s under a 300s ceiling), so the # nominal ~5m was a figure a busy catch could exceed, not a cap. Tests run - # sequentially, and the E2E job is capped at 215 timeout-minutes. A cascade + # sequentially, and the E2E job is capped at 265 timeout-minutes. A cascade # reddening many suites can therefore spend a large share of the job in # diagnostics. That is deliberate — the old ceiling # bought its speed by collecting nothing at all — and it is bounded: both legs # only approach their ceilings on a genuinely degraded cluster, a systemic # failure usually reds the install gate and skips the E2E step entirely, and # the report-upload steps are `if: always()`. That last point does NOT extend - # to a job timeout: a 215-minute cap ends the job `cancelled`, and the ~4m48s + # to a job timeout: a 265-minute cap ends the job `cancelled`, and the ~4m48s # grace window is consumed by the earlier `Collect report` step, so both # cozyreport.tgz and the job log are lost (measured on runs 29557003429 and # 29568562337). If this ever does start truncating runs, bound the number of diff --git a/hack/e2e-chainsaw/README.md b/hack/e2e-chainsaw/README.md index 98400bc55..ef5a50db4 100644 --- a/hack/e2e-chainsaw/README.md +++ b/hack/e2e-chainsaw/README.md @@ -26,7 +26,7 @@ A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Cha A parked `chainsaw-test.yaml.disabled` is none of these: the directory is then not a suite, is selected by nothing and cannot be allowlisted. -Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. +Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`, `observability-proxy`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. ## What Chainsaw buys over the BATS suite @@ -63,7 +63,7 @@ Declarative suites assert on `status.conditions` and concrete fields. Inherently ## Running locally -Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`. +Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`; `observability-proxy` needs `curl` and `nc`. ```bash # install chainsaw diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 939e8f69a..e6253c06a 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -91,7 +91,7 @@ spec: status: replicas: 1 --- -# ClickHouse Altinity backup/restore E2E — the two backup Tests below sit in +# ClickHouse Altinity backup/restore E2E — the three backup Tests below sit in # this suite alongside the plain `clickhouse` Test above. Chainsaw port of the # intended hack/e2e-apps/backup-clickhouse.bats (cozystack#2600), shaped like the # etcd/mariadb/postgres suites: @@ -107,8 +107,15 @@ spec: # driving the validated example scripts # (examples/backups/clickhouse). Also runs in # CI, in tenant-root. +# clickhouse-3-system-bucket-roundtrip +# — backup and in-place restore through the +# shipped cozy-default BackupClass and the +# platform system bucket, for a release +# whose Pod started before its credentials +# were projected. Runs in CI, in tenant-root. # -# Why the split: the two Tests answer different questions at different cost. The +# Why the split: the first two Tests answer different questions at different +# cost, and the third covers the system-bucket path neither of them takes. The # contracts Test needs no object store, so it stays in the default tenant-test # namespace next to the other app suites and is the seconds-long guard on the # chart wiring the Altinity strategy depends on — the sidecar container/port and @@ -377,3 +384,217 @@ spec: set -eu cd ../../.. NAMESPACE="$NAMESPACE" examples/backups/clickhouse/cleanup.sh +--- +# Backup and in-place restore through the shipped cozy-default BackupClass and +# the platform system bucket, for a ClickHouse whose Pod started before the +# platform projected cozy-backups-creds into its namespace. The sidecar has to +# pick those credentials up while running; until it could, the first BackupJob +# of such a release failed with "no EC2 IMDS role found" and kept failing until +# the Pod was restarted. +# +# Unlike clickhouse-2-backup-roundtrip this targets the system bucket as +# advertised, https://s3.example.org, so system-bucket-roundtrip.sh makes that +# name reachable from tenant-root for the duration of its own step and undoes +# it on exit: see its header for the two shared objects it touches and the +# cert-manager behaviour that relies on. Runs in tenant-root, where the system +# bucket's SeaweedFS and its Ingress controller sit. +# +# The precondition step deletes tenant-root's projected cozy-backups-creds, +# which any BackupJob that ran in tenant-root before this Test leaves there: +# the projector runs for each BackupJob whatever its strategy, and Chainsaw +# does not fix the order of the Tests in a suite, so the earlier one may come +# from this suite or another. It writes the Secret again on the next +# BackupJob, which is the moment under test. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: clickhouse-3-system-bucket-roundtrip +spec: + namespace: tenant-root + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: clickhouse-sysbucket + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: e2e-system-bucket-s3 + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: seaweedfs-s3-ingress-tls + - describe: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: ingress-seaweedfs-s3 + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: BackupJob + name: sysbucket-backup + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: RestoreJob + name: sysbucket-restore + - podLogs: + selector: app.kubernetes.io/instance=clickhouse-sysbucket + container: clickhouse-backup + tail: 200 + - podLogs: + selector: app=clickhouse-sysbucket-keeper + tail: 100 + steps: + - name: preconditions + try: + # The deletes below wait up to 270s in all (30+30+60+30+30+30+60), so a + # stuck one is named by its kubectl --timeout before the 5m ceiling kills + # the shell with no object named. A new delete needs its own --timeout. + - script: + timeout: 5m + content: | + set -eu + if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then + echo "Secret $NAMESPACE/seaweedfs-s3-ingress-tls already exists; this Test would have to replace it" >&2 + exit 1 + fi + if kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF 'BEGIN e2e-clickhouse-system-bucket'; then + echo "kube-system/coredns still carries the e2e-clickhouse-system-bucket block from an earlier run" >&2 + exit 1 + fi + # Fixed names in a persistent namespace: a leftover Succeeded job + # would satisfy the waits in the run step without a new attempt, and + # a leftover Backup of the BackupJob's name, recording an earlier + # run's archive, fails the new BackupJob. + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s + kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s + # Chainsaw reclaims only what its apply created, so objects a killed + # run left would be patched and kept. The Certificate goes before its + # Secret, which cert-manager issues again while the Certificate exists. + kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete pod e2e-system-bucket-probe --ignore-not-found --wait=true --timeout=60s + check: + ($error == null): true + + - name: create-certificate-and-probe + try: + - apply: + file: system-bucket-tls.yaml + - assert: + timeout: 2m + resource: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: e2e-system-bucket-s3 + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Pod + metadata: + name: e2e-system-bucket-probe + status: + (conditions[?type == 'Ready']): + - status: "True" + # The Certificate's Secret carries no owner reference, so it outlives the + # Certificate, and cert-manager issues it again while the Certificate + # exists. Chainsaw runs a step's cleanup ops after this step's cleaner has + # deleted the Certificate and waited for it. + cleanup: + - delete: + ref: + apiVersion: v1 + kind: Secret + name: e2e-system-bucket-s3-tls + + - name: create-clickhouse + try: + - apply: + file: clickhouse-system-bucket.yaml + # 11m: above the 600s install timeout cozystack-api gives this release; + # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. + - assert: + timeout: 11m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: clickhouse-sysbucket + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: chi-clickhouse-sysbucket-clickhouse-0-0 + status: + readyReplicas: 1 + # The Pod is running with no projected credentials: the state the first + # BackupJob of a fresh release meets. + - error: + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + name: cozy-backups-creds + + - name: run + try: + # Inner budgets of system-bucket-roundtrip.sh: + # 280s probe: a 240s deadline for CoreDNS picking up the rewrite and + # the Ingress serving the Test certificate, plus one last 30s + # attempt and its 10s pause + # 480s BackupJob to Succeeded, including the driver's 2m hold + # 480s RestoreJob to Succeeded; no hold by then, the stamp is + # already older than the backup's own 2m wait + # 90s the EXIT trap: Corefile replace plus a 60s Secret delete + # 1330s (22m10s) + # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so + # an inner wait is what fires and names what stalled. + - script: + timeout: 25m + content: | + set -eu + cd ../../.. + NAMESPACE="$NAMESPACE" hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh + check: + ($error == null): true + finally: + # The script's EXIT trap puts the Corefile and the Ingress TLS Secret + # back, but a step ceiling kills the shell before the trap runs. A no-op + # when the trap already did it. + - description: restore the host CoreDNS and the S3 Ingress TLS Secret + script: + timeout: 2m + content: | + set -eu + cd ../../.. + # Chainsaw runs this under sh; the helpers are bash. + NAMESPACE="$NAMESPACE" bash -euo pipefail -c ' + source examples/backups/clickhouse/00-helpers.sh + source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh + restore_shared_state + ' + # Chainsaw did not apply these: the jobs come from the script and the + # Backup from the controller, and deleting a BackupJob reclaims no Backup. + - description: system-bucket round-trip cleanup + script: + timeout: 3m + content: | + set -eu + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + # The driver names the Backup after the BackupJob; by name it is + # found even when status.backupRef was never written. + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml index b88e7dd6d..fb0979146 100644 --- a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml @@ -8,8 +8,8 @@ # backup.enabled=true with placeholder S3 coordinates is deliberate: the # sidecar runs `clickhouse-backup server`, which starts its HTTP API without # validating S3 (ALLOW_EMPTY_BACKUPS=true, lazy connect — the same reason the -# platform useSystemBucket flow marks the S3 secret keys optional so the Pod -# starts before any bucket is projected). So the Pod becomes Ready and the +# platform useSystemBucket flow can start the Pod before any bucket is +# projected). So the Pod becomes Ready and the # chart<->strategy contract is assertable with no reachable object store. The # real S3 backup/restore round-trip lives in clickhouse-2-backup-roundtrip, # which runs in tenant-root against the in-cluster SeaweedFS. diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml new file mode 100644 index 000000000..0ed1dd4a4 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml @@ -0,0 +1,30 @@ +# ClickHouse on the platform system bucket for the +# clickhouse-3-system-bucket-roundtrip Test. No S3 coordinates are given: the +# sidecar reads them from the clickhouse-backup.yml key the platform projects +# into cozy-backups-creds at the first BackupJob, after this Pod has started. +# backup.endpointCA trusts the SeaweedFS CA that signs the certificate the Test +# serves for s3.example.org. +apiVersion: apps.cozystack.io/v1alpha1 +kind: ClickHouse +metadata: + name: sysbucket +spec: + size: 5Gi + logStorageSize: 1Gi + shards: 1 + replicas: 1 + # The round-trip runs SQL against this server; at nano's 128Mi the first + # query OOM-kills it, so it takes the preset clickhouse-2 runs SQL on. + resourcesPreset: small + resources: {} + backup: + enabled: true + useSystemBucket: true + endpointCA: + name: e2e-system-bucket-s3-tls + key: ca.crt + clickhouseKeeper: + enabled: true + replicas: 1 + size: 1Gi + resourcesPreset: micro diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh new file mode 100755 index 000000000..c1c216389 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# Backup and in-place restore of the `sysbucket` ClickHouse through the +# cozy-default BackupClass, against the platform system bucket. Run from the +# repository root by the clickhouse-3-system-bucket-roundtrip Test, which has +# already started the release with no cozy-backups-creds in its namespace. +# +# The system bucket is addressed as https://s3.example.org, the S3 Ingress host +# the COSI driver advertises. In CI that name does not resolve and its ACME +# certificate is never issued, so for the duration of this script, and only of +# this script, two shared objects are changed and the EXIT trap below puts both +# back: +# +# - the S3 Ingress TLS Secret seaweedfs-s3-ingress-tls is created from the +# Test's own certificate (SeaweedFS CA, s3.example.org). It relies on +# cert-manager leaving a Secret's data alone while the Certificate that +# names it has no completed issuance, verified with cert-manager v1.20.2: +# a fresh ACME order would replace this certificate, and ingress-nginx +# would then serve one the sidecar does not trust, which fails the probe +# below rather than the backup. +# - the host CoreDNS Corefile gets one marked `rewrite name exact` line for +# s3.example.org. The host CoreDNS is Talos-managed and imports no snippet +# files, so the block goes into the live Corefile the way ouroboros writes +# its own; ouroboros is off in CI and no other suite uses this name. +set -euo pipefail + +source examples/backups/clickhouse/00-helpers.sh +source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh + +: "${NAMESPACE:?}" +CLICKHOUSE_NAME=sysbucket +BACKUPJOB_NAME=sysbucket-backup +RESTOREJOB_NAME=sysbucket-restore +CERT_SECRET=e2e-system-bucket-s3-tls +PROBE_POD=e2e-system-bucket-probe +DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local" +# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's +# reload plugin polls every 30s; the rest is the 30s answer cache. +PROBE_TIMEOUT=240 +# Each covers the driver's 2m hold after the credentials change plus the +# strategy Job itself. +JOB_TIMEOUT=480 + +workdir=$(mktemp -d) + +on_exit() { + local rc=$? + set +e + restore_shared_state || rc=1 + rm -rf "$workdir" + exit "$rc" +} + +if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -o name >/dev/null 2>&1; then + log_error "Secret ${NAMESPACE}/${INGRESS_SECRET} already exists; this Test serves its own certificate under that name and will not overwrite one it did not create" + exit 1 +fi +if corefile_has_block; then + log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand" + exit 1 +fi +trap on_exit EXIT + +log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \ + | jq --arg name "$INGRESS_SECRET" '{apiVersion, kind, type, + metadata: {name: $name, labels: {"cozystack.io/e2e-owner": "clickhouse-system-bucket"}}, + data: {"tls.crt": .data["tls.crt"], "tls.key": .data["tls.key"], "ca.crt": .data["ca.crt"]}}' \ + | kubectl -n "$NAMESPACE" create -f - + +log_step "Rewriting ${S3_HOST} to ${DNS_TARGET} in the host CoreDNS" +kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json" +jq -e '.data.Corefile | startswith(".:53 {\n")' "$workdir/coredns.json" >/dev/null \ + || { log_error "kube-system/coredns Corefile does not open with '.:53 {'; refusing to edit it"; exit 1; } +# The resourceVersion read above travels with the replace, so a concurrent +# writer makes it fail instead of being overwritten. +jq --arg block "${DNS_BEGIN}"$'\n'" rewrite name exact ${S3_HOST} ${DNS_TARGET}"$'\n'"${DNS_END}" \ + '.data.Corefile |= sub("^\\.:53 \\{\n"; ".:53 {\n" + $block + "\n")' \ + "$workdir/coredns.json" >"$workdir/coredns-new.json" +kubectl replace -f "$workdir/coredns-new.json" >/dev/null + +log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAMESPACE}" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt" +want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \ + | openssl x509 -noout -fingerprint -sha256) +deadline=$((SECONDS + PROBE_TIMEOUT)) +while true; do + # --cacert makes curl verify both the chain and the host name; %{certs} + # prints the chain it was served, leaf first. + if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ + "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \ + <"$workdir/ca.crt" 2>"$workdir/probe.err"); then + got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \ + | openssl x509 -noout -fingerprint -sha256) + if [ "$got" = "$want" ]; then + log_success "${S3_HOST} resolves in ${NAMESPACE} and serves the Test certificate" + break + fi + echo "served ${got}, want ${want}" >"$workdir/probe.err" + fi + if [ "$SECONDS" -ge "$deadline" ]; then + log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")" + kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true + kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true + exit 1 + fi + # TODO(e2e-replace-fixed-timeouts): no condition reports that the host + # CoreDNS answers with the rewrite and the Ingress serves the Test + # certificate, as seen from this namespace; poll until one does. + sleep 10 +done + +# A restarted Pod reads its credentials afresh at start, which is what the +# old environment-based sidecar needed; the round-trip proves the fix only if +# the sidecar that started without them is the one that serves both jobs. +sidecar_identity() { + kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \ + -o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}' +} +sidecar_before=$(sidecar_identity) +[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ + || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } + +# Restore takes the archive this run's Backup recorded; a row no earlier run +# wrote is what shows the data restored is that archive's. +sentinel="before-backup-$(date -u +%Y%m%dT%H%M%SZ)-${RANDOM}" +log_step "Writing the sentinel row ${sentinel}" +clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" +clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, '${sentinel}')" + +log_step "Backing up through cozy-default" +kubectl apply -f - <"$dir/coredns.json" || { rm -rf "$dir"; return 1; } + local begins + begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true) + if [ "$begins" = 0 ]; then + rm -rf "$dir" + return 0 + fi + if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then + log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand" + rm -rf "$dir" + return 1 + fi + jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \ + '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \ + "$dir/coredns.json" >"$dir/coredns-clean.json" + # Callers run this with errexit off, so a refused replace has to return. + if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then + log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand" + rm -rf "$dir" + return 1 + fi + log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns" + rm -rf "$dir" +} + +# Returns non-zero when either object could not be put back. +restore_shared_state() { + local rc=0 + remove_dns_block || rc=1 + # kubectl refuses a name together with a selector, so the label is + # looked up on its own and the name matched in what comes back. + local owned + if ! owned=$(kubectl -n "$NAMESPACE" get secret -l "$OWNER_LABEL" -o name); then + log_error "could not list the Secrets labelled ${OWNER_LABEL} in ${NAMESPACE}; delete ${INGRESS_SECRET} by hand if it carries that label" + rc=1 + elif printf '%s\n' "$owned" | grep -qxF "secret/$INGRESS_SECRET"; then + kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 + fi + return "$rc" +} diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml new file mode 100644 index 000000000..06203a299 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml @@ -0,0 +1,42 @@ +# Certificate and probe Pod for the clickhouse-3-system-bucket-roundtrip Test. +# +# The certificate is signed by the SeaweedFS CA already present in tenant-root, +# so its Secret carries that CA as ca.crt: the ClickHouse release trusts it via +# backup.endpointCA, and the round-trip script copies the same key pair under +# the name the platform's S3 Ingress serves for s3.example.org. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-system-bucket-s3 +spec: + secretName: e2e-system-bucket-s3-tls + dnsNames: + - s3.example.org + issuerRef: + kind: Issuer + name: seaweedfs-ca-issuer +--- +# Resolves and fetches s3.example.org from inside tenant-root, where the +# ClickHouse sidecar runs, so the script can prove the DNS rewrite and the +# served certificate before it spends minutes on a BackupJob. The image is the +# one hack/e2e-install-cozystack.bats pre-pulls; it carries curl but no openssl. +apiVersion: v1 +kind: Pod +metadata: + name: e2e-system-bucket-probe +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 0 + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: probe + image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47 + command: [sleep, "3600"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] diff --git a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml index 2a02dc06e..4e0c86535 100644 --- a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml @@ -35,7 +35,7 @@ # and the teardown. This is the third suite in a full run that brings up a # tenant cluster, next to kubernetes-latest and kubernetes-previous. A failure # that stalls worker join everywhere reddens all three, and their red paths -# together can exceed the E2E job's 215-minute cap, which cancels the job and +# together can exceed the E2E job's 265-minute cap, which cancels the job and # loses its log and report. The cap is sized for the red path of the # kubernetes pair only; this suite is not counted in it, and that is an # accepted risk. diff --git a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml index 75ee5a2fa..695425364 100644 --- a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml @@ -80,7 +80,8 @@ spec: # wait, all on the passing path. Raising the ceiling to cover them is not # available -- the guard in hack/run-kubernetes-serial-console_test.bats # holds COZY_OP_CEILING equal to BOTH suites' `timeout:`, and two suites - # at 79m plus their 22m teardowns exceed the job's own 215-minute cap. + # at 79m plus their 22m teardowns take 202m, which with the install and + # the rest of a full run does not fit the job's own 265-minute cap. # What is available is a measurement: on the green run of this branch # (33724179053) the whole Test took 569.72s against this 4020s op, with # kubernetes-previous, which carries no OIDC fold, at 470.19s. The diff --git a/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml b/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml new file mode 100644 index 000000000..576f6fa6d --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml @@ -0,0 +1,257 @@ +# Namespace logs through the observability proxy, as the console reaches +# them: every request goes through the console nginx (a port-forward to its +# Service), which routes the logs root to the proxy, which authenticates the +# bearer token, asks a SubjectAccessReview and forwards a rebuilt query to +# the tenant's log store. +# +# Two sibling tenants, A (logsa) and B (logsb), both under tenant-test, so +# both resolve to the root log store. A viewer of A reads A's logs and is +# refused B's; parameters and headers a client adds cannot widen the scope +# (the proxy forces the namespace filter and tenant 0:0); a live tail stays +# open and keeps delivering lines past 60s. +# +# Not covered here: the OIDC gatekeeper (oauth2-proxy) in front of nginx, +# because driving a Keycloak login needs a browser; and Fluent Bit writing +# real pod logs, because the root log store is off in e2e and this suite +# seeds its own (log-store.yaml). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: observability-proxy-logs +spec: + catch: + - events: {} + - podLogs: + namespace: cozy-observability-proxy + selector: app.kubernetes.io/name=observability-proxy + tail: 100 + - podLogs: + namespace: tenant-root + selector: app.kubernetes.io/name=e2e-observability-log-store + tail: 100 + - podLogs: + namespace: cozy-dashboard + selector: app.kubernetes.io/name=console + tail: 100 + steps: + - name: proxy-available + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: observability-proxy + namespace: cozy-observability-proxy + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: tenants + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: logsa + spec: {} + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: logsb + spec: {} + # 16m: tenant-rd's 15m install timeout plus the 60s start margin; see + # "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. Waited so + # that cleanup does not uninstall a release whose install still runs. + - assert: + timeout: 16m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: tenant-logsa + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 16m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: tenant-logsb + status: + (conditions[?type == 'Ready']): + - status: "True" + # The label names the store the proxy reads; the seeded store below + # stands in for the root one, so both namespaces must resolve to it. + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-logsa + labels: + namespace.cozystack.io/monitoring: tenant-root + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-logsb + labels: + namespace.cozystack.io/monitoring: tenant-root + + - name: viewer-and-store + try: + # Applying over a real root store would rewrite its Service; refuse to. + - error: + resource: + apiVersion: v1 + kind: Service + metadata: + name: vlselect-generic + namespace: tenant-root + - apply: + file: viewer.yaml + - apply: + file: log-store.yaml + - assert: + timeout: 3m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: e2e-observability-log-store + namespace: tenant-root + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: seed-logs + try: + - script: + # 30s forward + 3 x 15s ingests + 3 x 30s visibility polls, each + # capped on its own, under a 4m op. + timeout: 4m + content: | + set -eu + . ./lib.sh + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT + forward tenant-root vlselect-generic 19471:9471 + ingest tenant-test-logsa e2e-marker-a + ingest tenant-test-logsb e2e-marker-b + # Same namespace, another VictoriaLogs tenant: reachable only if a + # client AccountID header got through. + ingest tenant-test-logsa e2e-marker-account7 7 + # All three must be searchable, or an absence checked later + # proves nothing. + for m in e2e-marker-a:0 e2e-marker-b:0 e2e-marker-account7:7; do + timeout 30 sh -ec "until curl --silent --fail --header 'AccountID: ${m#*:}' '$STORE/select/logsql/query?query=${m%:*}' | grep -q ${m%:*}; do sleep 1; done" + done + + - name: namespace-scope + try: + - script: + # 30s forward + one 30s token request + 10 calls capped at 30s. + timeout: 8m + content: | + set -eu + . ./lib.sh + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT + forward cozy-dashboard cozy-dashboard-console 18080:8080 + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) + auth="Authorization: Bearer $token" + + code=$(call "$LOGS_A/query?query=*") + expect 401 "$code" "no token" + + code=$(call "$LOGS_A/query?query=*" --header "$auth") + expect 200 "$code" "viewer reads A" + grep -q e2e-marker-a "$body" || { echo "FAIL: A's own line is missing" >&2; cat "$body" >&2; exit 1; } + absent e2e-marker-b "viewer reads A" + absent e2e-marker-account7 "viewer reads A" + + code=$(call "$LOGS_B/query?query=*" --header "$auth") + expect 403 "$code" "viewer reads B" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query=*' \ + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}' \ + --data-urlencode 'extra_filters=kubernetes_namespace_name:tenant-test-logsb') + expect 200 "$code" "client extra filters in the query string" + absent e2e-marker-b "client extra filters in the query string" + + code=$(call "$LOGS_A/query" --header "$auth" \ + --data-urlencode 'query=*' \ + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}') + expect 200 "$code" "client extra filters in a POST form" + absent e2e-marker-b "client extra filters in a POST form" + + code=$(call "$LOGS_A/query?query=*" --header "$auth" --header "AccountID: 7" --header "ProjectID: 0") + expect 200 "$code" "client AccountID header" + absent e2e-marker-account7 "client AccountID header" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query={kubernetes_namespace_name="tenant-test-logsb"}') + expect 200 "$code" "B's stream named in the query" + absent e2e-marker-b "B's stream named in the query" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query=options(ignore_global_time_filter=true) *') + expect 400 "$code" "query options" + + # nginx routes on the normalised path, which is B's logs root, and + # forwards the raw one; the proxy must refuse it, not read B. + code=$(call "$LOGS_A/../../../../tenant-test-logsb/logs/select/logsql/query?query=*" --path-as-is --header "$auth") + expect 400 "$code" "dot segments toward B" + + code=$(call "$LOGS_A/delete?query=*" --header "$auth") + expect 404 "$code" "an endpoint outside the allowlist" + + - name: tail-outlives-a-minute + try: + - script: + # 2 x 30s forwards + 30s token + 3 x 15s ingests + the 70s hold + + # the 30s delivery poll; the tail itself is cut at 150s. + timeout: 6m + content: | + set -eu + . ./lib.sh + tail_out=$(mktemp) + trap 'kill "${tail_pid:-}" 2>/dev/null || true; for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body" "$tail_out"' EXIT + forward tenant-root vlselect-generic 19471:9471 + forward cozy-dashboard cozy-dashboard-console 18080:8080 + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) + + # The tail is not subscribed yet when the next ingest runs, so it + # replays the last minute to include the early line. + timeout 150 curl --silent --no-buffer --header "Authorization: Bearer $token" \ + "$CONSOLE$LOGS_A/tail?query=*&start_offset=1m" > "$tail_out" & + tail_pid=$! + ingest tenant-test-logsa e2e-tail-early + ingest tenant-test-logsb e2e-tail-b + # TODO(e2e-replace-fixed-timeouts): this hold stands in for a + # duration assertion the harness lacks; the property under test is + # that a tail survives a minute with the client sending nothing. + sleep 70 + if ! kill -0 "$tail_pid" 2>/dev/null; then + echo "FAIL: the tail closed before 70s" >&2 + cat "$tail_out" >&2 + exit 1 + fi + ingest tenant-test-logsa e2e-tail-late + # vlselect re-reads every second, five seconds behind now. + timeout 30 sh -ec "until grep -q e2e-tail-late '$tail_out'; do sleep 1; done" || { + echo "FAIL: a line written at 70s did not reach the open tail" >&2 + cat "$tail_out" >&2 + exit 1 + } + grep -q e2e-tail-early "$tail_out" || { echo "FAIL: the early line is missing from the tail" >&2; exit 1; } + if grep -q e2e-tail-b "$tail_out"; then + echo "FAIL: the tail of A delivered B's line" >&2 + exit 1 + fi diff --git a/hack/e2e-chainsaw/observability-proxy/lib.sh b/hack/e2e-chainsaw/observability-proxy/lib.sh new file mode 100644 index 000000000..9601cb826 --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/lib.sh @@ -0,0 +1,58 @@ +# shellcheck shell=sh +# Shared by the script steps of this suite, whose cwd is this directory. +# Every call carries its own cap so a hang fails at a known budget rather +# than at the op ceiling (docs/agents/e2e-testing.md, section 8). + +k() { timeout 30 kubectl "$@"; } + +STORE=http://localhost:19471 +CONSOLE=http://localhost:18080 +# shellcheck disable=SC2034 # read by the steps that source this file +LOGS_A=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsa/logs/select/logsql +# shellcheck disable=SC2034 +LOGS_B=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsb/logs/select/logsql + +# forward :. The caller's EXIT trap +# kills every pid in pf_pids; the forward gets 30s to start listening. +forward() { + kubectl port-forward -n "$1" "service/$2" "$3" >/dev/null 2>&1 & + pf_pids="${pf_pids:-} $!" + timeout 30 sh -ec "until nc -z localhost ${3%%:*}; do sleep 1; done" +} + +# ingest [account]: one entry shaped as Fluent Bit +# writes it, with kubernetes_namespace_name as a stream field. +ingest() { + printf '{"kubernetes_namespace_name":"%s","log":"%s"}\n' "$1" "$2" | + timeout 15 curl --silent --show-error --fail --data-binary @- \ + --header 'Content-Type: application/json' --header "AccountID: ${3:-0}" --header "ProjectID: 0" \ + "$STORE/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log" +} + +# call [curl args...]: a request through the console nginx, the way +# the browser sends it. Prints the status; the body lands in $body. +body=$(mktemp) +call() { + path=$1 + shift + timeout 30 curl --silent --show-error --output "$body" --write-out '%{http_code}' "$@" "$CONSOLE$path" +} + +# expect +expect() { + if [ "$1" != "$2" ]; then + echo "FAIL: $3: HTTP $2, want $1" >&2 + cat "$body" >&2 + exit 1 + fi + echo "ok: $3: HTTP $2" +} + +# absent : the last body must not carry the marker. +absent() { + if grep -q "$1" "$body"; then + echo "FAIL: $2: the response carries $1, which is outside the scope" >&2 + cat "$body" >&2 + exit 1 + fi +} diff --git a/hack/e2e-chainsaw/observability-proxy/log-store.yaml b/hack/e2e-chainsaw/observability-proxy/log-store.yaml new file mode 100644 index 000000000..491dc667c --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/log-store.yaml @@ -0,0 +1,71 @@ +# A single-node VictoriaLogs under the name the proxy dials for a namespace +# whose logs belong to the root tenant: vlselect-generic.tenant-root, port +# 9471. The e2e install leaves root monitoring off, so the real one does not +# exist; the suite seeds this one itself instead of waiting for Fluent Bit, +# which keeps the test about the proxy's scope rather than log collection. +# The version is the one victoria-metrics-operator v0.68.4 deploys. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: e2e-observability-log-store + namespace: tenant-root +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: e2e-observability-log-store + template: + metadata: + labels: + app.kubernetes.io/name: e2e-observability-log-store + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: victoria-logs + image: docker.io/victoriametrics/victoria-logs:v1.47.0 + args: + - -httpListenAddr=:9471 + - -storageDataPath=/storage + - -retentionPeriod=1d + ports: + - name: http + containerPort: 9471 + readinessProbe: + httpGet: + path: /health + port: http + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: storage + mountPath: /storage + volumes: + - name: storage + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: vlselect-generic + namespace: tenant-root +spec: + selector: + app.kubernetes.io/name: e2e-observability-log-store + ports: + - name: http + port: 9471 + targetPort: http diff --git a/hack/e2e-chainsaw/observability-proxy/viewer.yaml b/hack/e2e-chainsaw/observability-proxy/viewer.yaml new file mode 100644 index 000000000..0563ad4a0 --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/viewer.yaml @@ -0,0 +1,22 @@ +# The view level of tenant A and nothing else: a ServiceAccount of tenant A +# bound to cozy:tenant:view in A's namespace only. Its token is a real bearer +# token, so the proxy's TokenReview and SubjectAccessReview run unmocked. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: logs-viewer + namespace: tenant-test-logsa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: logs-viewer + namespace: tenant-test-logsa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:tenant:view +subjects: +- kind: ServiceAccount + name: logs-viewer + namespace: tenant-test-logsa diff --git a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml index 170c0ce36..bc0bc1c98 100644 --- a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml @@ -347,3 +347,148 @@ spec: set -eu cd ../../.. NAMESPACE="$NAMESPACE" examples/backups/postgres/cleanup.sh +--- +# A system database that a version before the refusal took over is handed back +# by the next init-job run, and only what this chart marked changes hands. +# +# The helm-unittest and bats suites check the rendered SQL against a fake psql; +# only a real PostgreSQL can say that the predicates select what they claim. The +# state an older version left is rebuilt by hand as the superuser, since the +# chart now refuses to produce it: +# * postgres is commented as managed and owned by a postgres_admin role that +# carries the cozystack:managed mark, as the chart left it: it must come back +# to the superuser, and the role must be dropped; +# * template1 is commented as managed, but its template1_admin carries only the +# role comment, which a CREATEROLE holder can forge, and not the mark: the +# comment must be cleared and everything that role owns or was granted must +# stay as it is, in template1 and in a database cloned from it. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: postgres-3-system-databases +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: postgres-sysdb + - podLogs: + selector: job-name=postgres-sysdb-init-job + tail: 200 + steps: + - name: create-postgres + try: + - apply: + file: postgres-sysdb.yaml + # 11m: above the 600s install timeout cozystack-api gives this release; + # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. + - assert: + timeout: 11m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: postgres-sysdb + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: postgres-sysdb-init-job + status: + (conditions[?type == 'Complete']): + - status: "True" + + - name: rebuild-taken-over-system-databases + try: + - script: + timeout: 3m + content: | + set -eu + pod=$(kubectl -n "$NAMESPACE" get pod \ + -l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1) + [ -n "$pod" ] || { echo "no primary pod for postgres-sysdb" >&2; exit 1; } + sql() { kubectl -n "$NAMESPACE" exec -i "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -d "$1"; } + sql postgres <<'SQL' + CREATE ROLE postgres_admin NOLOGIN; + COMMENT ON ROLE postgres_admin IS 'role managed by helm'; + GRANT "cozystack:managed" TO postgres_admin WITH INHERIT FALSE, SET FALSE; + COMMENT ON DATABASE postgres IS 'database managed by helm'; + ALTER DATABASE postgres OWNER TO postgres_admin; + CREATE ROLE template1_admin NOLOGIN; + COMMENT ON ROLE template1_admin IS 'role managed by helm'; + CREATE ROLE template1_readonly NOLOGIN; + COMMENT ON DATABASE template1 IS 'database managed by helm'; + ALTER DATABASE template1 OWNER TO template1_admin; + SQL + sql template1 <<'SQL' + ALTER SCHEMA public OWNER TO template1_admin; + CREATE TABLE public.kept (id int); + ALTER TABLE public.kept OWNER TO template1_admin; + GRANT SELECT ON public.kept TO template1_readonly; + SQL + sql postgres <<'SQL' + CREATE DATABASE sysdb_clone TEMPLATE template1; + SQL + # The next run must be a new Job: remove the completed one so the + # assert below cannot match it. + kubectl -n "$NAMESPACE" delete job postgres-sysdb-init-job --wait=true + # Declaring a user is enough for an upgrade, and runs the init-job. + kubectl -n "$NAMESPACE" patch postgres.apps.cozystack.io sysdb --type merge \ + -p '{"spec":{"users":{"sysdbuser":{},"sysdbprobe":{}}}}' + check: + ($error == null): true + - assert: + timeout: 11m + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: postgres-sysdb-init-job + status: + (conditions[?type == 'Complete']): + - status: "True" + + - name: verify-release + try: + - script: + timeout: 2m + content: | + set -eu + pod=$(kubectl -n "$NAMESPACE" get pod \ + -l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1) + q() { kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -tA -d "$1" -c "$2"; } + fail=0 + expect() { + got=$(q "$1" "$3") + if [ "$got" != "$4" ]; then + echo "FAIL: $2: got '$got', want '$4'" >&2 + fail=1 + fi + } + comment="SELECT coalesce(shobj_description(oid, 'pg_database'), '') FROM pg_database WHERE datname = " + owner="SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = " + expect postgres "postgres comment cleared" "$comment'postgres'" '' + expect postgres "postgres back to the superuser" "$owner'postgres'" postgres + expect postgres "marked postgres_admin dropped" "SELECT count(*) FROM pg_roles WHERE rolname = 'postgres_admin'" 0 + expect postgres "template1 comment cleared" "$comment'template1'" '' + expect postgres "unmarked template1_admin still exists" "SELECT count(*) FROM pg_roles WHERE rolname = 'template1_admin'" 1 + expect postgres "template1 still owned by the unmarked role" "$owner'template1'" template1_admin + for db in template1 sysdb_clone; do + expect "$db" "$db public still owned by the unmarked role" \ + "SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = 'public'" template1_admin + expect "$db" "$db kept table still owned by the unmarked role" \ + "SELECT tableowner FROM pg_tables WHERE schemaname = 'public' AND tablename = 'kept'" template1_admin + expect "$db" "$db grant to the unmarked readonly role kept" \ + "SELECT has_table_privilege('template1_readonly', 'public.kept', 'SELECT')" t + done + exit "$fail" + check: + ($error == null): true +# The Postgres CR was applied by chainsaw; deleting it removes the whole +# cluster, the databases and roles built above included. diff --git a/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml b/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml new file mode 100644 index 000000000..975dc6a51 --- /dev/null +++ b/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml @@ -0,0 +1,18 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Postgres +metadata: + name: sysdb +spec: + external: false + size: 1Gi + replicas: 1 + storageClass: "" + quorum: + minSyncReplicas: 0 + maxSyncReplicas: 0 + users: + sysdbuser: {} + databases: + appdb: {} + resources: {} + resourcesPreset: "nano" diff --git a/hack/monitoring-agents-vm-cleanup-hook.bats b/hack/monitoring-agents-vm-cleanup-hook.bats new file mode 100644 index 000000000..290a91314 --- /dev/null +++ b/hack/monitoring-agents-vm-cleanup-hook.bats @@ -0,0 +1,312 @@ +#!/usr/bin/env bats +# Behavioural tests for the monitoring-agents pre-delete hook: +# +# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml +# +# The script is extracted from the rendered chart and RUN against a fake +# kubectl that keeps the VictoriaMetrics objects in a state file, with fake +# `date` and `sleep` so the 60s grace period passes in an instant. The +# helm-unittest suite in packages/system/monitoring-agents/tests/ +# uninstall_test.yaml matches the script text instead, and stayed green when the +# ownership condition was removed, when the finalizer patch was made a no-op and +# when the polling condition was inverted. Each of those turns this file red. +# +# What is pinned here: +# * only objects whose Helm annotations name this release, by name and +# namespace both, are deleted or patched: an object of another release, of +# a same-named release in another namespace, or one Helm does not own, is +# left alone; +# * with the operator gone, the hook waits before touching finalizers, then +# clears them, and nothing of this release is left behind; +# * with the operator alive, it finalizes the objects itself and the hook +# never patches a finalizer; +# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a +# failing pre-delete hook would itself block the uninstall. +# +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper + +MA_CHART=packages/system/monitoring-agents +MA_RELEASE=monitoring-agents +MA_NAMESPACE=cozy-monitoring + +# render_hook +# Extract the named Job's shell script from the rendered chart into . The +# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every +# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left +# in last-command position has its status discarded. +render_hook() { + helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2" + [ -s "$2" ] || return 1 +} + +# write_fakes +# Emit fake `kubectl`, `date` and `sleep` into . +# +# kubectl reads and writes $STATE, one object per line as +# kind|release-name|release-namespace|namespace|name, where the two release +# fields are the Helm annotations (empty when Helm does not own the object). +# `get` prints this kind's objects in the hook's jsonpath format, or fails for a +# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete` +# removes the object only when $OPERATOR_LIVE is 1, standing for an operator +# that finalizes it; otherwise the object stays, Terminating. `patch` removes it +# only when the payload actually clears the finalizers. Every call goes to $KLOG. +# +# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty +# of polls, and records each time it returns in $KLOG; sleep records itself, +# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus +# shows how much fake time passed before each patch. +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. +write_fakes() { + cat > "$1/kubectl" <<'KEOF' +#!/bin/sh +echo "kubectl $*" >> "$KLOG" +drop() { + awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new" + mv "$STATE.new" "$STATE"; } +case "$1" in + get) + kind=${2%%.*} + case " ${MISSING_KINDS:-} " in + *" $kind "*) + echo "error: the server doesn't have a resource type \"$kind\"" >&2 + exit 1 ;; + esac + awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE" + exit 0 ;; + -n) + ns=$2 verb=$3 kind=${4%%.*} name=$5 + case "$verb" in + delete) + [ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name" + echo "$kind \"$name\" deleted" + exit 0 ;; + patch) + case "$*" in + *'"finalizers":null'*) drop "$kind" "$ns" "$name" ;; + esac + echo "$kind \"$name\" patched" + exit 0 ;; + esac ;; +esac +echo "fake kubectl: unexpected call: $*" >&2 +exit 64 +KEOF + cat > "$1/date" <<'KEOF' +#!/bin/sh +c=$(cat "$CLOCK" 2>/dev/null || echo 0) +echo $((c + 1)) > "$CLOCK" +echo "date $((1000000 + c))" >> "$KLOG" +echo $((1000000 + c)) +KEOF + cat > "$1/sleep" <<'KEOF' +#!/bin/sh +echo "sleep $*" >> "$KLOG" +if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then + c=$(cat "$CLOCK" 2>/dev/null || echo 0) + echo $((c + $1)) > "$CLOCK" +fi +KEOF + chmod +x "$1/kubectl" "$1/date" "$1/sleep" +} + +# write_state +# Three objects of this release in two namespaces; a VMRule of another release; +# a VMRule whose release name matches but whose release namespace does not, so +# only the pair of annotations tells it apart; and a VMServiceScrape that +# carries no Helm annotations. +write_state() { + cat > "$1" < +# Run the rendered hook with the fakes first on PATH; its status lands in $rc. +run_hook() { + export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock" + : > "$KLOG" + rc=0 + PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$? +} + +@test "a render that produces no script fails instead of passing as a no-op" { + # Guards the guard: every test below runs whatever render_hook produced, so an + # empty render would satisfy the exit-0 assertions without executing a line of + # the hook. + tmp=$(mktemp -d) + if render_hook no-such-job "$tmp/hook.sh"; then + echo "FAIL: render_hook reported success for a Job name the chart never emits" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + for obj in "vmagents.operator.victoriametrics.com vmagent" \ + "vmrules.operator.victoriametrics.com alerts-node" \ + "vmservicescrapes.operator.victoriametrics.com operator"; do + if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then + echo "FAIL: $obj was not deleted" + cat "$tmp/calls" + false + fi + done + # The objects of this release must be gone, which the fake only does for a + # patch that really clears the finalizers. + if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then + echo "FAIL: objects of this release are still there, Terminating" + cat "$tmp/state" "$tmp/calls" + false + fi + # Another release's object, one of a same-named release in another + # namespace, and an object Helm does not own are never touched. + if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then + echo "FAIL: the hook touched an object this release does not own" + cat "$tmp/calls" + false + fi + if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then + echo "FAIL: the three objects this release does not own should remain" + cat "$tmp/state" + false + fi + # The operator gets its chance first: the hook polls and sleeps before the + # first patch, and does not stop waiting before the 60s are up. + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + if [ -z "$first_patch" ]; then + echo "FAIL: no finalizer was cleared" + cat "$tmp/calls" + false + fi + # The first date call sets the deadline; the last one before the first patch + # is the time the hook gave up waiting. + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then + echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers before its Job's deadline" { + # The grace-period test above keeps sleep free so a shortened wait shows; here + # sleep costs its argument, so a poll interval long enough to overshoot the + # Job's activeDeadlineSeconds fails instead of passing. + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -) + export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1 + run_hook "$tmp" + unset SLEEP_ADVANCES + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then + echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook leaves the finalizers to a live operator" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=1 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then + echo "FAIL: the VMAgent was not deleted" + cat "$tmp/calls" + false + fi + if grep -q ' patch ' "$tmp/calls"; then + echo "FAIL: the hook cleared finalizers the live operator was handling" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then + echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone" + cat "$tmp/state" "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" { + tmp=$(mktemp -d) + write_fakes "$tmp" + printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \ + "$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with nothing to clean" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -E -q ' (delete|patch) ' "$tmp/calls"; then + echo "FAIL: the hook deleted or patched with nothing of this release left" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} diff --git a/hack/observability-proxy-seed.bats b/hack/observability-proxy-seed.bats new file mode 100644 index 000000000..26caf2091 --- /dev/null +++ b/hack/observability-proxy-seed.bats @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +prepare_seed_capture() { + strict_setup + seed_tmp=$(mktemp -d) + seed_old_path=$PATH + mkdir -p "$seed_tmp/bin" + export SEED_CAPTURE="$seed_tmp/seed" + export PATH="$seed_tmp/bin:$PATH" + cat > "$seed_tmp/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -eu +printf '%s\n' "$@" > "$SEED_CAPTURE.args" +cat > "$SEED_CAPTURE.body" +if ! grep --fixed-strings --line-regexp --quiet 'Content-Type: application/json' "$SEED_CAPTURE.args"; then + echo 'seed request is missing its JSON content type' >&2 + exit 22 +fi +EOF + chmod +x "$seed_tmp/bin/curl" +} + +assert_seed_request() { + local account=$1 + prepare_seed_capture + # shellcheck source=e2e-chainsaw/observability-proxy/lib.sh + . "$REPO_ROOT/hack/e2e-chainsaw/observability-proxy/lib.sh" + if [ "$account" = 0 ]; then + ingest tenant-test-logsa e2e-marker-a + else + ingest tenant-test-logsa e2e-marker-a "$account" + fi + grep --fixed-strings --line-regexp --quiet "AccountID: $account" "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet 'ProjectID: 0' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet 'http://localhost:19471/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet -- '--data-binary' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet '@-' "$SEED_CAPTURE.args" + printf '%s\n' '{"kubernetes_namespace_name":"tenant-test-logsa","log":"e2e-marker-a"}' > "$SEED_CAPTURE.expected" + cmp "$SEED_CAPTURE.expected" "$SEED_CAPTURE.body" + rm "$body" "$SEED_CAPTURE.args" "$SEED_CAPTURE.body" "$SEED_CAPTURE.expected" "$seed_tmp/bin/curl" + rmdir "$seed_tmp/bin" "$seed_tmp" + export PATH="$seed_old_path" +} + +@test "observability seeds send JSON without consuming the body as form data" { + assert_seed_request 0 +} + +@test "observability seeds preserve an explicit VictoriaLogs account" { + assert_seed_request 7 +} diff --git a/hack/postgres-release-system-databases.bats b/hack/postgres-release-system-databases.bats new file mode 100644 index 000000000..688f20df5 --- /dev/null +++ b/hack/postgres-release-system-databases.bats @@ -0,0 +1,261 @@ +#!/usr/bin/env bats +# Behavioural tests for the "release system databases" pass of the Postgres +# init script: +# +# packages/apps/postgres/templates/init-script.yaml +# +# The pass is cut out of the rendered script and RUN against a fake psql that +# records which database each session connects to. The helm-unittest suite in +# packages/apps/postgres/tests/system_databases_test.yaml matches the SQL text, +# and stayed green when the template0 guard was inverted: the rendered pass then +# connected to template0, which accepts no connection, and skipped postgres and +# template1, which are the two it exists to hand back. +# +# What is pinned here: +# * no session ever connects to template0; +# * postgres and template1 are each connected to and released; +# * every taken-over system database, template0 included, has its comment +# cleared; +# * releasing template1 visits every other connectable database, for the +# trigger it passed on to them, and a failed lookup of those databases +# stops the pass before the comment that selects template1 is cleared; +# * the session that releases a system database hands its public schema +# and the database itself back only from a marked _admin or +# _readonly owner (or cozystack:orphaned), before the reassignment, +# an inherited public only from a marked template1_admin, and touches +# only the roles that carry the +# "cozystack:managed" mark, not a role that merely has one of the names. +# +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper + +PG_CHART=packages/apps/postgres + +# render_pass +# Extract the release pass, from its banner to the next one, out of the rendered +# init script. The non-empty check must be an early `return 1`: hack/cozytest.sh +# rewrites every line matching ^}$ into `return 0` plus `}`, helpers included, +# so a check left in last-command position has its status discarded. +render_pass() { + helm template pg-test "$PG_CHART" --namespace tenant-test --set-json '_cluster={"scheduling":{}}' 2>/dev/null \ + | yq eval 'select(.metadata.name == "pg-test-init-script") | .stringData["init.sh"]' - \ + | awk '/^echo "== release system databases"/ { f = 1 } /^echo "== delete databases"/ { f = 0 } f' > "$1" + grep -q 'SYSTEM_DBS=' "$1" || return 1 +} + +# write_fake_psql +# Emit a fake psql. A `-c` query answers from $SYSTEM_DBS for the system +# database lookup and from $OTHER_DBS for the connectable-database lookup, +# which exits with $CLONE_LOOKUP_FAILS instead when that is set. A +# session fed on stdin logs "connect " ("connect -" without --dbname) +# followed by its first line and every -v db= value, to $PLOG, and its whole +# input under a "-- session db=" header to $PLOG.sql. +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. +write_fake_psql() { + cat > "$1/psql" <<'PEOF' +#!/bin/sh +dbname=- query= vdb= +while [ $# -gt 0 ]; do + case "$1" in + --dbname) dbname=$2; shift ;; + -c) query=$2; shift ;; + -v) case "$2" in db=*) vdb=${2#db=} ;; esac; shift ;; + esac + shift +done +if [ -n "$query" ]; then + case "$query" in + *"d.datname IN ('postgres', 'template0', 'template1')"*) printf '%s\n' $SYSTEM_DBS ;; + *datallowconn*) + if [ -n "${CLONE_LOOKUP_FAILS:-}" ]; then + echo "fake psql: connectable-database lookup failed" >&2 + exit "$CLONE_LOOKUP_FAILS" + fi + printf '%s\n' $OTHER_DBS ;; + *) echo "fake psql: unexpected query: $query" >&2; exit 64 ;; + esac + exit 0 +fi +sql=$(cat) +echo "connect $dbname db=$vdb $(printf '%s\n' "$sql" | head -n 1)" >> "$PLOG" +printf -- '-- session %s db=%s\n%s\n' "$dbname" "$vdb" "$sql" >> "$PLOG.sql" +PEOF + chmod +x "$1/psql" +} + +# run_pass +# Run the extracted pass with the fake psql first on PATH; status in $rc. +run_pass() { + export PLOG="$1/calls" + : > "$PLOG" + rc=0 + PATH="$1:$PATH" sh -e "$1/pass.sh" > "$1/out" 2> "$1/err" || rc=$? +} + +@test "a render that produces no release pass fails instead of passing as a no-op" { + tmp=$(mktemp -d) + if PG_CHART=packages/apps/no-such-chart render_pass "$tmp/pass.sh"; then + echo "FAIL: render_pass reported success for a chart that does not exist" + false + fi + rm -rf "$tmp" +} + +@test "release pass connects to postgres and template1, never to template0" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres template0 template1" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q '^connect template0 ' "$tmp/calls"; then + echo "FAIL: the pass connected to template0, which accepts no connection" + cat "$tmp/calls" + false + fi + for db in postgres template1; do + if ! grep -q "^connect $db db=$db DROP EVENT TRIGGER IF EXISTS trigger_auto_grant;" "$tmp/calls"; then + echo "FAIL: $db was not connected to and released" + cat "$tmp/calls" + false + fi + done + for db in postgres template0 template1; do + if ! grep -q "^connect - db=$db COMMENT ON DATABASE :\"db\" IS NULL;" "$tmp/calls"; then + echo "FAIL: the comment on $db was not cleared" + cat "$tmp/calls" + false + fi + done + rm -rf "$tmp" +} + +@test "release session hands public back and touches only marked roles" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres" OTHER_DBS="postgres" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + awk '/^-- session / { f = ($0 == "-- session postgres db=postgres") } f' "$tmp/calls.sql" > "$tmp/release.sql" + # Each statement must end on its whole predicate: a clause appended after the + # mark (an OR, say) would select an unmarked role again. + reassign_where="WHERE EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid JOIN pg_roles u ON u.oid = m.member WHERE g.rolname = 'cozystack:managed' AND u.rolname = r)\\gexec" + owner_marked="(o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\\gexec" + public_where="WHERE n.nspname = 'public' AND $owner_marked" + database_where="WHERE d.datname = :'db' AND $owner_marked" + public_line=$(grep -n 'ALTER SCHEMA public OWNER TO' "$tmp/release.sql" | head -1) + database_line=$(grep -n 'ALTER DATABASE' "$tmp/release.sql" | head -1) + reassign_line=$(grep -n 'REASSIGN OWNED BY' "$tmp/release.sql" | head -1) + if [ "${public_line##*"$public_where"}" != "" ] || [ -z "$public_line" ]; then + echo "FAIL: the release session does not hand public back only from a marked _admin or _readonly owner" + cat "$tmp/release.sql" + false + fi + if [ "${database_line##*"$database_where"}" != "" ] || [ -z "$database_line" ]; then + echo "FAIL: the release session does not take the database back only from a marked owner" + cat "$tmp/release.sql" + false + fi + if [ "${reassign_line##*"$reassign_where"}" != "" ] || [ -z "$reassign_line" ]; then + echo "FAIL: the release session does not reassign only roles carrying the cozystack:managed mark" + cat "$tmp/release.sql" + false + fi + # public and the database must be handed back before REASSIGN OWNED would + # move them to cozystack:orphaned. + if [ "${public_line%%:*}" -ge "${reassign_line%%:*}" ] || [ "${database_line%%:*}" -ge "${reassign_line%%:*}" ]; then + echo "FAIL: public or the database is handed back after the reassignment has already moved it" + cat "$tmp/release.sql" + false + fi + rm -rf "$tmp" +} + +@test "release pass of template1 visits every other connectable database" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="template1" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + for db in postgres app; do + if ! grep -q "^connect $db db= SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant'" "$tmp/calls"; then + echo "FAIL: $db was not checked for the trigger inherited from template1" + cat "$tmp/calls" + false + fi + done + # An inherited public changes hands only from a template1_admin the chart + # marked; an owner that merely has the name keeps it. + inherited_where="WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid)\\gexec" + inherited_line=$(awk '/^-- session / { f = ($0 == "-- session app db=") } f' "$tmp/calls.sql" | grep 'ALTER SCHEMA public OWNER TO' | head -1) + if [ -z "$inherited_line" ] || [ "${inherited_line##*"$inherited_where"}" != "" ]; then + echo "FAIL: an inherited public is handed back without checking that a marked template1_admin owns it" + cat "$tmp/calls.sql" + false + fi + rm -rf "$tmp" +} + +@test "release pass of template1 stops before clearing its comment when the clone lookup fails" { + # Clearing the comment is what stops later runs from selecting template1, so + # a lookup failure that let the pass go on would skip the clone repair for + # good. + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="template1" OTHER_DBS="postgres app" CLONE_LOOKUP_FAILS=73 + run_pass "$tmp" + unset CLONE_LOOKUP_FAILS + + if [ "$rc" -ne 73 ]; then + echo "FAIL: the pass exited $rc instead of stopping on the failed lookup (73)" + cat "$tmp/out" "$tmp/err" "$tmp/calls" + false + fi + if grep -q 'COMMENT ON DATABASE' "$tmp/calls"; then + echo "FAIL: the comment on template1 was cleared although its clones were never visited" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "release pass of postgres alone leaves the other databases alone" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q '^connect app ' "$tmp/calls"; then + echo "FAIL: the pass visited app although template1 was not taken over" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index a0868ec38..f0c597583 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -2,11 +2,11 @@ package backupcontroller import ( - "bytes" "context" "encoding/json" "fmt" "maps" + "regexp" "strings" "time" @@ -57,6 +57,12 @@ const ( // archive relies on. altinityArchiveTimeLayout = "20060102T150405Z" + // altinityArchiveRunTimeLayout and altinityArchiveUIDLength shape the + // archive name of a run; see altinityArchiveName for why they are this + // short. + altinityArchiveRunTimeLayout = "20060102150405" + altinityArchiveUIDLength = 12 + // Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG / // Velero strategy cadence so behaviour is uniform across drivers. altinityPollInterval = 5 * time.Second @@ -85,6 +91,76 @@ func validateAltinityApplicationRef(ref corev1.TypedLocalObjectReference) error return nil } +// altinityCredentialsPropagation bounds how long kubelet takes to rewrite +// the mounted cozy-backups-creds in the ClickHouse Pod after it changes: +// its sync period (a minute by default) plus the Secret cache. About 75s was +// measured; the margin covers the rest. +const altinityCredentialsPropagation = 2 * time.Minute + +// credentialsPropagationWait returns how long to hold back creating the +// strategy Job named jobName in namespace, so that the clickhouse-backup +// sidecar, which reads the projected credentials from a mounted file, sees +// their latest content. A Job that already exists, or a namespace without +// projected credentials, does not wait. +func credentialsPropagationWait(ctx context.Context, c client.Client, secretName, namespace, jobName string) (time.Duration, error) { + if secretName == "" { + return 0, nil + } + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: jobName}, &batchv1.Job{}); err == nil { + return 0, nil + } else if !apierrors.IsNotFound(err) { + return 0, err + } + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secretName}, secret); err != nil { + if apierrors.IsNotFound(err) { + return 0, nil + } + return 0, err + } + raw := secret.Annotations[credentialsUpdatedAtAnnotation] + if raw == "" { + return 0, nil + } + // The projector rewrites the stamp only when the content changes, so a + // stamp this hold cannot use stays as it is: holding on it would block + // every Job in the namespace for good. + updatedAt, err := time.Parse(time.RFC3339, raw) + if err != nil { + getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation) + return 0, nil + } + if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) { + getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw) + return 0, nil + } + return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil +} + +// clearCredentialsWait drops the Ready condition credentialsWaitCondition +// left behind, once the job no longer waits. It reports whether it did. +func clearCredentialsWait(conditions *[]metav1.Condition) bool { + ready := apimeta.FindStatusCondition(*conditions, "Ready") + if ready == nil || ready.Reason != credentialsWaitReason { + return false + } + return apimeta.RemoveStatusCondition(conditions, "Ready") +} + +const credentialsWaitReason = "WaitingForCredentials" + +// credentialsWaitCondition is the Ready condition of a job held back by +// credentialsPropagationWait. +func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition { + return metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: credentialsWaitReason, + Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar", + wait.Round(time.Second), secretName), + } +} + // jobNameForBackupJob returns the deterministic batch/v1.Job name used for // the backup pod that drives a BackupJob. Idempotency relies on this being // pure: a retried reconcile must hit the same Job name. @@ -117,11 +193,30 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { } // altinityArchiveName mints the archive name for one BackupJob from its -// persisted StartedAt, so every reconcile and every Pod retry of the Job -// writes the same archive. It keeps the clickhouse-- -// shape, because a Backup that records no archive is restored by the -// timestamp in that shape. +// persisted StartedAt and its UID, so every reconcile and every Pod retry of +// the Job writes the same archive and no other run writes it. StartedAt has +// one-second resolution, and two Plans on one schedule start their runs in +// the same second; the strategy takes a finished upload of its archive as an +// earlier attempt of its own run, so a shared name would let one run report +// the other's archive. +// +// The name is no longer than altinityArchiveNameBeforeUIDs, because it sits in +// every object key, a key is capped at 1024 bytes, and the legacy +// s3PathOverride prefix has no length limit of its own: the UID takes the room +// that dropping "clickhouse-", "T" and "Z" frees. It goes after the timestamp +// so names still sort chronologically, and the restore of a Backup that +// records no archive, which accepts only clickhouse--, +// never picks one of these. func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { + uid := strings.ReplaceAll(string(j.UID), "-", "") + uid = uid[:min(len(uid), altinityArchiveUIDLength)] + return fmt.Sprintf("%s-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveRunTimeLayout), uid) +} + +// altinityArchiveNameBeforeUIDs is the name a controller that did not append +// the UID gave the same BackupJob, which a Job it created before an upgrade +// still writes. +func altinityArchiveNameBeforeUIDs(j *backupsv1alpha1.BackupJob) string { return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) } @@ -129,13 +224,16 @@ func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { // the archive name. A Job created by a controller that predates archive // names, or rendered from a strategy that names its own archives, wrote an // archive under another name, and recording this one would point the Backup -// at an archive that does not exist. +// at an archive that does not exist. The name has to stand on its own: the +// name without the UID starts the name with it, and a strategy may append to +// either. func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool { spec, err := json.Marshal(job.Spec.Template.Spec) if err != nil { return false } - return bytes.Contains(spec, []byte(archive)) + whole := regexp.MustCompile(`(^|[^A-Za-z0-9._-])` + regexp.QuoteMeta(archive) + `($|[^A-Za-z0-9._-])`) + return whole.Match(spec) } // renderAltinityTemplate runs the strategy's PodTemplateSpec through the @@ -265,6 +363,18 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, j.Namespace, jobNameForBackupJob(j)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&j.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityJob(ctx, j, j.Namespace, jobNameForBackupJob(j), altinityModeBackup, map[string]string{ @@ -283,7 +393,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return ctrl.Result{}, nil } if !altinityJobCarriesArchive(batchJob, archive) { - archive = "" + archive = altinityArchiveNameBeforeUIDs(j) + if !altinityJobCarriesArchive(batchJob, archive) { + archive = "" + } } artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive) if err != nil { @@ -312,6 +425,11 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, message) default: + if clearCredentialsWait(&j.Status.Conditions) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } @@ -681,6 +799,18 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, targetNamespace, jobNameForRestoreJob(restoreJob)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&restoreJob.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob), altinityModeRestore, map[string]string{ @@ -717,6 +847,11 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, message) default: + if clearCredentialsWait(&restoreJob.Status.Conditions) { + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index e5f1e1695..a9022ceb6 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -3,22 +3,26 @@ package backupcontroller import ( "context" + "errors" "strings" "testing" "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -730,14 +734,66 @@ func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) j := &backupsv1alpha1.BackupJob{ - Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, - Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } - if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + if got, want := altinityArchiveName(j), "ch-test-20260304050607-0f3c9a1e5b7d"; got != want { t.Errorf("altinityArchiveName = %q, want %q", got, want) } } +// TestAltinityArchiveName_NoLongerThanTheNameWithoutUID pins the S3 key +// budget: the archive name is part of every object key, a key may not exceed +// 1024 bytes, and the legacy s3PathOverride prefix is unbounded, so a prefix +// that fit the name without the UID must still fit this one. +func TestAltinityArchiveName_NoLongerThanTheNameWithoutUID(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + for n := 1; n <= 63; n++ { + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef(strings.Repeat("a", n))}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + name, legacy := altinityArchiveName(j), altinityArchiveNameBeforeUIDs(j) + if len(name) > len(legacy) { + t.Errorf("application name of %d characters: archive %q is longer than %q", n, name, legacy) + } + // The manifest key of a prefix that left the name without the UID + // exactly at the limit. + prefix := strings.Repeat("p", 1024-len("/"+legacy+"/metadata.json")) + if key := prefix + "/" + name + "/metadata.json"; len(key) > 1024 { + t.Errorf("application name of %d characters: manifest key is %d bytes", n, len(key)) + } + } +} + +// TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond pins that two +// BackupJobs of one application started within the same second, as two Plans +// on the same schedule are, write different archives. With a shared name, the +// strategy takes the other run's finished upload for an earlier attempt of its +// own and records an archive missing what changed since that upload. +func TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + run := func(name, uid string) *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + } + daily := altinityArchiveName(run("daily-1", "11111111-1111-1111-1111-111111111111")) + weekly := altinityArchiveName(run("weekly-1", "22222222-2222-2222-2222-222222222222")) + if daily == weekly { + t.Fatalf("two runs started in the same second share archive %q", daily) + } + for _, name := range []string{daily, weekly} { + if !strings.HasPrefix(name, "ch-test-20260304050607-") { + t.Errorf("archive %q does not start with the application and start time, so names stop sorting chronologically", name) + } + } +} + // TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob // through Job creation and completion in separate reconciles. The archive // name has to come out of the persisted StartedAt both times: the Pod is @@ -748,7 +804,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -765,7 +821,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "ch-test-20260304050607-testbjuid" if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { t.Fatalf("reconcileAltinity() creating the Job: %v", err) @@ -801,6 +857,90 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } } +// TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs covers a backup +// Job created by a controller that named archives without the BackupJob's UID +// and still running when the controller is upgraded: the Job is not +// re-rendered, so its archive carries the old name, and the Backup has to +// record that name rather than none. +func TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs(t *testing.T) { + const legacy = "clickhouse-ch-test-20260304T050607Z" + cases := []struct { + name string + archive string // archive the Job's Pod was rendered with + want string + }{ + {name: "the name without the UID", archive: legacy, want: legacy}, + {name: "a longer name that starts with it", archive: legacy + "-copy", want: ""}, + {name: "a longer name that ends with it", archive: "copy-" + legacy, want: ""}, + {name: "a longer name that continues it after a period", archive: legacy + ".copy", want: ""}, + {name: "a longer name that leads into it with a period", archive: "copy." + legacy, want: ""}, + {name: "a longer name that continues it after an underscore", archive: legacy + "_copy", want: ""}, + {name: "a longer name that leads into it with an underscore", archive: "copy_" + legacy, want: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := recordAltinityArchiveOfJobRenderedWith(t, tc.archive); got != tc.want { + t.Errorf("Backup records archive %q, want %q", got, tc.want) + } + }) + } +} + +// recordAltinityArchiveOfJobRenderedWith completes a backup Job whose Pod +// carries the given archive in place of the one the controller rendered, and +// returns the archive the resulting Backup records. +func recordAltinityArchiveOfJobRenderedWith(t *testing.T, archive string) string { + t.Helper() + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + job.Spec.Template.Spec.Containers[0].Args[1] = "--archive=" + archive + if err := r.Update(ctx, job); err != nil { + t.Fatalf("render the backup Job with archive %q: %v", archive, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + return created.Spec.DriverMetadata[altinityArchiveMetaKey] +} + // TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose // Pod never carried the driver's archive name: one created before the // upgrade that introduced the names, or rendered from a strategy that names @@ -869,13 +1009,13 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { // A Backup recording this run's own archive is the one an earlier reconcile // created before its status write was lost, and is adopted. func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { - const thisRun = "clickhouse-ch-test-20260305T050607Z" + const thisRun = "ch-test-20260305050607-testbjuid" cases := []struct { name string existing string // archive the existing Backup records; empty for none wantSucceeded bool }{ - {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records another archive", existing: "ch-test-20260304050607-earlierbjuid"}, {name: "earlier run's Backup records no archive", existing: ""}, {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, } @@ -885,7 +1025,7 @@ func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -1032,3 +1172,539 @@ func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) { }) } } + +func projectedCredentials(namespace string, updatedAt time.Time) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: updatedAt.UTC().Format(time.RFC3339)}, + }, + } +} + +// The ClickHouse sidecar reads the projected credentials from a mounted +// file that kubelet rewrites only some time after the Secret changes, so a +// strategy Job started right after a projection would reach a sidecar with +// no or stale credentials. +func TestReconcileAltinity_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + cases := []struct { + name string + secret *corev1.Secret + jobExists bool + wantWait bool + }{ + {name: "credentials changed just now", secret: projectedCredentials("tenant-test", time.Now()), wantWait: true}, + {name: "credentials changed long ago", secret: projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))}, + {name: "no projected credentials", secret: nil}, + {name: "the Job already runs", secret: projectedCredentials("tenant-test", time.Now()), jobExists: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + objs := []client.Object{backupJob, newAltinityStrategy("clickhouse-strategy")} + if tc.secret != nil { + objs = append(objs, tc.secret) + } + if tc.jobExists { + objs = append(objs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}}) + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), clientfake.NewClientBuilder().WithObjects(objs...)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, err := r.reconcileAltinity(ctx, j, resolved) + if err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + ready := meta.FindStatusCondition(j.Status.Conditions, "Ready") + if tc.wantWait { + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created %d time(s) before the credentials could reach the sidecar", len(jobs.Items)) + } + if res.RequeueAfter <= 0 || res.RequeueAfter > altinityCredentialsPropagation { + t.Errorf("RequeueAfter = %v, want within (0, %v]", res.RequeueAfter, altinityCredentialsPropagation) + } + if ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } + return + } + if len(jobs.Items) != 1 { + t.Errorf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + if ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("Ready = %+v, the job should not wait", ready) + } + }) + } +} + +func TestReconcileAltinityRestore_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now()))) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + res, err := rr.reconcileAltinityRestore(ctx, rj, backup) + if err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created before the credentials could reach the sidecar") + } + if res.RequeueAfter <= 0 { + t.Errorf("RequeueAfter = %v, want a wait", res.RequeueAfter) + } + if ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } +} + +// Without credentials projection configured there is no Secret to wait +// for; client-go refuses a Get with an empty name, so it must not be asked. +func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy")). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } +} + +func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp}, + }, + } +} + +func waitingForCredentials() []metav1.Condition { + return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)} +} + +// Once the hold is over and the Job runs, a Ready condition still saying the +// job is waiting for credentials describes a state the job has left. +func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()}, + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.BackupJob{})) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + stored := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + +func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning, + Conditions: waitingForCredentials()}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.RestoreJob{})) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + stored := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + +// The stamp is only ever rewritten when the projected content changes, so a +// value the hold cannot use must not hold the job: an unreadable one would +// fail every reconcile, and one far in the future would wait again on each. +// A stamp a little ahead, as a skewed clock writes it, still waits, but only +// until that stamp plus the propagation time. +func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) { + cases := []struct { + name string + stamp string + wantMin time.Duration + wantMax time.Duration + }{ + {name: "unreadable", stamp: "garbage"}, + {name: "far in the future", stamp: "3000-01-01T00:00:00Z"}, + {name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339), + wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build() + wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job") + if err != nil { + t.Fatalf("credentialsPropagationWait: %v", err) + } + if wait < tc.wantMin || wait > tc.wantMax { + t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax) + } + }) + } +} + +func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) { + conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}} + if clearCredentialsWait(&conditions) { + t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady") + } + if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" { + t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready) + } +} + +// A read the hold cannot complete must not release the Job: starting it +// before the credentials reach the sidecar is the failure the hold prevents. +func TestReconcileAltinity_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err == nil { + t.Errorf("reconcileAltinity returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created although the %s could not be read", tc.name) + } + }) + } +} + +func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err == nil { + t.Errorf("reconcileAltinityRestore returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created although the %s could not be read", tc.name) + } + }) + } +} + +// A backup held for the credentials keeps the archive name it was given from +// its persisted StartedAt: the held reconciles write the job's status, and the +// Pod created after the hold, and the Backup recorded after it, must still +// name the same archive, which is stamped no later than the Backup's takenAt. +func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + secret := projectedCredentials("tenant-test", time.Now()) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityArchiveStrategy("clickhouse-strategy"), secret)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + const want = "ch-test-20260304050607-0f3c9a1e5b7d" + reconcile := func(step string) { + t.Helper() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("%s: get BackupJob: %v", step, err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("%s: reconcileAltinity: %v", step, err) + } + } + + reconcile("held") + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); !apierrors.IsNotFound(err) { + t.Fatalf("the Job exists while the credentials wait should hold it: %v", err) + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + secret.Annotations[credentialsUpdatedAtAnnotation] = time.Now().Add(-altinityCredentialsPropagation - time.Second).UTC().Format(time.RFC3339) + if err := r.Update(ctx, secret); err != nil { + t.Fatalf("age the credentials stamp: %v", err) + } + reconcile("after the wait") + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want { + t.Fatalf("backup Pod was rendered with %q after the wait, want --archive=%s", got, want) + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + reconcile("completed") + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want { + t.Errorf("Backup records archive %q, want %q", got, want) + } + if stamp := created.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout); stamp < "20260304T050607Z" { + t.Errorf("Backup takenAt %s is earlier than its archive's stamp", stamp) + } +} diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 941b00323..6376c8e87 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -36,6 +36,7 @@ type s3ObjectInfo struct { type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) + StatObject(ctx context.Context, bucket, key string) (int64, error) } // s3Connection is what a driver resolves from its rendered strategy to @@ -53,6 +54,9 @@ type s3Connection struct { // what the writer of the backup trusted, no more. CAPEM []byte CAOnly bool + // InsecureSkipVerify mirrors an explicit admin opt-in a strategy + // already applies to its own uploads; it is never a fallback. + InsecureSkipVerify bool } // objectStore reads back what a backup wrote: the Secrets it was written @@ -104,14 +108,18 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { Region: conn.Region, Secure: conn.Secure, } - if conn.Secure && len(conn.CAPEM) > 0 { + if conn.Secure && (len(conn.CAPEM) > 0 || conn.InsecureSkipVerify) { // Start from minio's transport to keep its proxy and timeout settings. tr, err := minio.DefaultTransport(true) if err != nil { return nil, err } - if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { - return nil, err + if len(conn.CAPEM) > 0 { + if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { + return nil, err + } + } else { + tr.TLSClientConfig.InsecureSkipVerify = true } opts.Transport = tr } @@ -148,6 +156,18 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) StatObject(ctx context.Context, bucket, key string) (int64, error) { + info, err := l.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + if err != nil { + return 0, err + } + // minio-go reports -1 when the response carries no Content-Length. + if info.Size < 0 { + return 0, fmt.Errorf("object %s/%s: the endpoint reported no size", bucket, key) + } + return info.Size, nil +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 3d9c35b23..aff417344 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -14,6 +14,7 @@ import ( "math/big" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -37,6 +38,16 @@ type fakeS3Lister struct { bucket, prefix string recursive bool calls int + + statSize int64 + statErr error + statKey string +} + +func (f *fakeS3Lister) StatObject(_ context.Context, bucket, key string) (int64, error) { + f.calls++ + f.bucket, f.statKey = bucket, key + return f.statSize, f.statErr } func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { @@ -167,6 +178,11 @@ func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s return nil, ctx.Err() } +func (blockingLister) StatObject(ctx context.Context, _, _ string) (int64, error) { + <-ctx.Done() + return 0, ctx.Err() +} + // withShortProbeTimeout shortens the probe timeout and fails the test if fn // is not back well before the default timeout would have expired. func withShortProbeTimeout(t *testing.T, fn func()) { @@ -233,6 +249,76 @@ func TestFailBackupArtifact(t *testing.T) { }) } +// unrelatedCAPEM mints a CA that signed nothing an httptest server serves: +// every httptest server shares one certificate. +func unrelatedCAPEM(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +// TestMinioObjectLister_EnforcesTheCA pins that a configured CA is the +// trust the client verifies with, whether alone or on top of the system +// roots: an endpoint whose certificate it did not sign is refused. +func TestMinioObjectLister_EnforcesTheCA(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1") + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + })) + defer srv.Close() + // minio-go retries a TLS failure with backoff; a refusal needs no more + // than the first attempt. + stat := func(ca []byte, caOnly bool) error { + l, err := newMinioObjectLister(s3Connection{ + Endpoint: strings.TrimPrefix(srv.URL, "https://"), Secure: true, + AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1", + CAPEM: ca, CAOnly: caOnly, + }) + if err != nil { + t.Fatalf("newMinioObjectLister: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = l.StatObject(ctx, "cozy-backups", "k") + return err + } + signer := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + for _, caOnly := range []bool{true, false} { + if err := stat(signer, caOnly); err != nil { + t.Errorf("CAOnly=%v: the CA that signed the endpoint: %v", caOnly, err) + } + if err := stat(unrelatedCAPEM(t), caOnly); err == nil { + t.Errorf("CAOnly=%v: StatObject succeeded against a certificate the CA did not sign, want a TLS error", caOnly) + } + } +} + +// TestMinioObjectLister_StatWithoutSize pins that a HEAD answer without +// Content-Length, which minio-go turns into size -1, is not a size. +func TestMinioObjectLister_StatWithoutSize(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + })) + defer srv.Close() + l, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1"}) + if err != nil { + t.Fatalf("newMinioObjectLister: %v", err) + } + if size, err := l.StatObject(context.Background(), "cozy-backups", "k"); err == nil { + t.Errorf("StatObject = %d, nil; want an error for a response without Content-Length", size) + } +} + // TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system // roots" and "the CA alone". The system pool holds the endpoint's // certificate, and the configured CA is an unrelated one, so the endpoint @@ -243,18 +329,7 @@ func TestEndpointTrustPool_CAOnly(t *testing.T) { system := x509.NewCertPool() system.AddCert(endpoint.Certificate()) - // Every httptest server shares one certificate, so the unrelated CA is - // minted here. - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} - der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) - if err != nil { - t.Fatal(err) - } - unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + unrelatedCA := unrelatedCAPEM(t) verifies := func(caOnly bool) bool { pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone()) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..6741fbaf6 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -183,6 +183,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // deletes the object and WAITS for that delete before the Backup is // removed, so no object is orphaned (see cleanupKafkaBackup). return r.cleanupKafkaBackup(ctx, backup) + case strategyv1alpha1.KubernetesStrategyKind: + // Owns two artifacts: the Velero backup and the etcd dump object. + return r.cleanupKubernetesBackup(ctx, backup) case strategyv1alpha1.VeleroStrategyKind: return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup) default: diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index b86278649..15ef65db4 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -214,6 +214,8 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return r.reconcileRedis(ctx, j, resolved) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafka(ctx, j, resolved) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetes(ctx, j, resolved) default: logger.V(1).Info("BackupJob resolved StrategyRef.Kind not supported, skipping", "backupjob", j.Name, @@ -240,6 +242,7 @@ func supportedBackupStrategyKinds() []string { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } } @@ -315,14 +318,6 @@ func (r *BackupJobReconciler) handleProjectionError(ctx context.Context, j *back } // markBackupJobFailed records a terminal Failed phase on the BackupJob. -// -// Coupling note: a failure that fires before reconcileCNPG's StartedAt block -// has set backupJob.Status.StartedAt leaves StartedAt nil. The CNPG path's -// cnpgDefaultBackupDeadline check only fires once StartedAt is set, so an -// early-failure retry that reaches the StartedAt block restarts the deadline -// budget from the retry's StartedAt - intentional, since retries are -// user-driven and a fresh budget is what users expect after correcting the -// cause of the previous failure. func (r *BackupJobReconciler) markBackupJobFailed(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, message string) (ctrl.Result, error) { logger := getLogger(ctx) now := metav1.Now() diff --git a/internal/backupcontroller/backupjob_controller_test.go b/internal/backupcontroller/backupjob_controller_test.go index f802fe374..994654987 100644 --- a/internal/backupcontroller/backupjob_controller_test.go +++ b/internal/backupcontroller/backupjob_controller_test.go @@ -45,6 +45,7 @@ func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } sort.Strings(got) sort.Strings(want) diff --git a/internal/backupcontroller/backupjob_restore_lock.go b/internal/backupcontroller/backupjob_restore_lock.go index 06f165285..6b662cad5 100644 --- a/internal/backupcontroller/backupjob_restore_lock.go +++ b/internal/backupcontroller/backupjob_restore_lock.go @@ -32,13 +32,13 @@ const ConditionReasonRestoreInProgress = "RestoreInProgress" // // A backup must not start against an application a restore is rewriting. It // would capture a half-restored state at best, and on the CNPG path it does -// worse: every BackupJob reconcile re-applies the Cluster's barman plugin, and -// one landing between the restore re-rendering the Cluster with its new -// serverName and the plugin being attached can put the restored cluster back -// on the source's WAL prefix, where its recovery fails with "Expected empty -// archive". Plan-created runs are the ones that hit this, since nothing else -// looks at restores before creating a BackupJob. -func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) { +// worse: a BackupJob applies the Cluster's barman plugin before it creates +// its Backup, and an apply landing between the restore re-rendering the +// Cluster with its new serverName and the plugin being attached can put the +// restored cluster back on the source's WAL prefix, where its recovery fails +// with "Expected empty archive". Plan-created runs are the ones that hit +// this, since nothing else looks at restores before creating a BackupJob. +func activeRestoreTargeting(ctx context.Context, c client.Reader, namespace string, app corev1.TypedLocalObjectReference) (string, error) { app = NormalizeApplicationRef(app) list := &backupsv1alpha1.RestoreJobList{} if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil { @@ -80,7 +80,7 @@ func activeRestoreTargeting(ctx context.Context, c client.Client, namespace stri // reads options.targetNamespace, so the option is looked at only when the // Backup's strategy is Velero — the CNPG and Job-based drivers ignore it and // restore in place. -func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) { +func restoreJobTarget(ctx context.Context, c client.Reader, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) { backup := &backupsv1alpha1.Backup{} if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil { if client.IgnoreNotFound(err) == nil { @@ -130,13 +130,14 @@ func ptrString(s *string) string { return *s } -// holdForRestore keeps a BackupJob that has not started yet waiting while a -// restore writes into its application. It stays non-terminal, so a scheduled -// run still happens once the restore ends, only later than its slot. +// holdForRestore keeps a BackupJob waiting while a restore writes into its +// application: one that has not started yet, or a CNPG one about to write +// its WAL archive destination. It stays non-terminal, so a scheduled run +// still happens once the restore ends, only later than its slot. // // A failed status write is returned so the manager retries it with backoff. -// The job is held either way, since nothing past the gate ran, but a hold -// nobody can see reads as a run stuck for no reason. +// The job is held either way, since it writes nothing past the hold, but a +// hold nobody can see reads as a run stuck for no reason. func (r *BackupJobReconciler) holdForRestore(ctx context.Context, j *backupsv1alpha1.BackupJob, restore string) (ctrl.Result, error) { if j.Status.Phase == backupsv1alpha1.BackupJobPhaseEmpty { j.Status.Phase = backupsv1alpha1.BackupJobPhasePending diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 85e55e8c2..bdf9c5d7b 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -284,7 +284,7 @@ func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { r := reconcilerWithLister(t, l) r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } @@ -311,7 +311,7 @@ func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} - if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil { + if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")); err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } if l.calls != 0 { @@ -331,7 +331,7 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testi store := newCNPGStrategyTestClient(t, backup) r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} - got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil || got == nil || got.Name != "bj-race" { t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err) } @@ -349,7 +349,7 @@ func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T store := newCNPGStrategyTestClient(t, backup) r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} - _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if !errors.Is(err, errBackupNotReadBack) { t.Errorf("err = %v, want errBackupNotReadBack", err) } @@ -409,7 +409,7 @@ func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) { cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} - got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil || got == nil { t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) } @@ -451,3 +451,128 @@ func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) { t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) } } + +// The probe measures the base backup where it was written: the destination +// recorded on the cnpg.io/Backup when it was created, not the strategy as +// it reads by the time the backup completes. +func TestCreateCNPGBackupArtifact_SizeProbeReadsTheRecordedDestination(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-moved"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + oldCreds := "creds-before-edit" + written := cnpgDestination{DestinationPath: "s3://old-bkt/before/edit/", EndpointURL: "https://old.example:9000", S3SecretRef: &oldCreds} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "before/edit/srv/base/20261021T100000/data.tar.gz", Size: 42}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", oldCreds)) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.bucket != "old-bkt" || l.prefix != "before/edit/srv/base/20261021T100000/" { + t.Errorf("listed %s/%s, want the recorded destination old-bkt/before/edit/srv/base/20261021T100000/", l.bucket, l.prefix) + } + if l.conn.Endpoint != "old.example:9000" { + t.Errorf("probed endpoint %q, want the recorded old.example:9000", l.conn.Endpoint) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 42 { + t.Errorf("Artifact = %#v, want SizeBytes 42", got.Status.Artifact) + } +} + +// A backup recorded without a credentials Secret is not measured with the +// Secret the strategy names now: that Secret belongs to another location. +func TestCreateCNPGBackupArtifact_SizeProbeSkipsARecordWithoutCredentials(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-nocreds"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + written := cnpgDestination{DestinationPath: "s3://bkt/tenant/pg/", EndpointURL: "https://s3.example:8443"} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times with the strategy's Secret for a backup recorded without one", l.calls) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 0 { + t.Errorf("Artifact = %#v, want no size", got.Status.Artifact) + } +} + +// probeRecordedBackup runs createCNPGBackupArtifact for a backup recorded as +// written with credentials Secret recordedSecret, against rendered, and +// returns the size it measured and the connection it measured through. +func probeRecordedBackup(t *testing.T, rendered *strategyv1alpha1.CNPGTemplate, recordedSecret *corev1.Secret) (int64, s3Connection) { + t.Helper() + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-probe"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + secretName := recordedSecret.Name + written := cnpgDestination{DestinationPath: "s3://bkt/tenant/pg/", EndpointURL: "https://s3.example:8443", S3SecretRef: &secretName} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 42}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, recordedSecret) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if got.Status.Artifact == nil { + t.Fatalf("no artifact") + } + return got.Status.Artifact.SizeBytes, l.conn +} + +// A strategy that has since dropped its credentials still measures a backup +// recorded with a Secret, through that Secret and its default keys. +func TestCreateCNPGBackupArtifact_SizeProbeOfARecordedSecretUnderAStrategyWithoutCredentials(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials = nil + size, conn := probeRecordedBackup(t, &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *store}, s3CredsSecret("tenant", "creds-recorded")) + if size != 42 { + t.Errorf("SizeBytes = %d, want 42 measured through the recorded Secret", size) + } + if conn.AccessKey != "AKID" { + t.Errorf("probed with access key %q, want the recorded Secret's AKID", conn.AccessKey) + } +} + +// The record names the Secret, not its keys: the strategy's key names stay, +// so a strategy with its own key names still reads the recorded Secret. +func TestCreateCNPGBackupArtifact_SizeProbeKeepsTheStrategysSecretKeys(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials.AccessKeyIDKey = "id" + store.S3Credentials.SecretAccessKeyKey = "key" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds-recorded"}, + Data: map[string][]byte{"id": []byte("CUSTOMID"), "key": []byte("custom-secret")}, + } + size, conn := probeRecordedBackup(t, &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *store}, secret) + if size != 42 { + t.Errorf("SizeBytes = %d, want 42 measured with the strategy's key names", size) + } + if conn.AccessKey != "CUSTOMID" { + t.Errorf("probed with access key %q, want CUSTOMID read under the strategy's key name", conn.AccessKey) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 819efc10f..e43deff33 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -15,6 +15,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" @@ -175,13 +176,17 @@ const ( // against an unreachable target reproduces the same FATAL. cnpgRecoveryMaxInspectPods = 5 - // Cap on the wall-clock time a BackupJob can spend observing a - // cnpg.io/Backup stuck in phase=failed before the driver gives up and - // marks the BackupJob Failed. CNPG can transition out of "failed" on - // internal retries (e.g. transient instance-manager restart), so we - // don't fail on the first observation. But without this cap, a - // permanently-broken Backup would pin the BackupJob in Running forever, - // blocking later runs and wedging the Plan-controller queue. + // Cap on the wall-clock time a BackupJob spends before its + // cnpg.io/Backup exists on each of two waits. The plugin rollout is + // counted from the attach, the later of the PluginRolledOut condition and + // the Cluster's cnpgPluginAttachedAtAnnotation, or from the release + // recorded in DestinationWriteHeld when that is later. Retrying the WAL + // archive destination write is counted from the first failure since the + // last successful write or the last wait for another backup or a + // restore. A Backup that exists is not capped: CloudNativePG keeps a + // healthy one pending while another backup of the same Cluster runs, or + // while it restarts one on another instance, so no fixed deadline tells + // those apart from a stuck one. cnpgDefaultBackupDeadline = 30 * time.Minute cnpgClusterHealthyPhase = "Cluster in healthy state" @@ -192,11 +197,69 @@ const ( // instance pod spec (the plugin injects its sidecar), so CNPG rolls every // instance, and a cnpg.io/Backup created meanwhile fails with "requested // plugin is not available" or "instance manager was restarted during - // backup" and never recovers. False holds the Backup back; its - // LastTransitionTime, written before the plugin is attached, is what the - // instance pods must postdate (see cnpgPluginRolloutPending). + // backup" and never recovers. False holds the Backup back; the instance + // pods must postdate its LastTransitionTime and the Cluster's + // cnpgPluginAttachedAtAnnotation (see cnpgReasonAttachingPlugin and + // cnpgPluginRolloutPending). backupCondPluginRolledOut = "PluginRolledOut" + // Reason of the PluginRolledOut latch recorded before the write that + // attaches the plugin. Every write of this job that attaches it replaces + // the condition with its own time, which retries can put long after the + // latch. + cnpgReasonAttachingPlugin = "AttachingPlugin" + + // Condition Type recorded while writing the WAL archive destination keeps + // failing. Its LastTransitionTime marks the first failure, so the retry + // budget is not spent by time the job waited for another backup of the + // app, or for a restore of it, before it tried. + backupCondDestinationApplied = "DestinationApplied" + + // Condition Type recorded while the job is held off writing the WAL + // archive destination, by another backup of the app in flight or by a + // restore of it, and set False when released. The release time restarts + // the plugin rollout deadline, which that wait must not spend; the + // instance pods still have to postdate the rollout's own start. + backupCondDestinationWriteHeld = "DestinationWriteHeld" + + // Annotation on a cnpg.io/Backup carrying, as JSON, the WAL archive + // destination written by the pass that created it: the destination + // path, the endpoint URL, the name of the credentials Secret and the + // serverName the Cluster archived under. + cnpgWrittenDestinationAnnotation = "backups.cozystack.io/cnpg-written-destination" + + // Annotation on a cnpg.io/Cluster carrying, as RFC 3339, when a backup + // job attached the barman-cloud plugin. It is written by the same apply + // that attaches the plugin, so unlike the PluginRolledOut condition it + // cannot be lost to a failed BackupJob status write after the attach. + cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at" + + // Annotation on the ObjectStore carrying, as RFC 3339, when a backup job + // last changed its configuration. It is written by the apply that makes + // the change, so it cannot be lost apart from it. + cnpgDestinationChangedAtAnnotation = "backups.cozystack.io/destination-changed-at" + + // Annotation on the ObjectStore naming the BackupJob whose apply last + // wrote it. Written by that apply, it holds the destination for that job + // while it runs and has no cnpg.io/Backup of its own yet; from then on + // the queue sees that Backup instead. + cnpgDestinationWrittenByAnnotation = "backups.cozystack.io/destination-written-by" + + // How long after an ObjectStore configuration change a Backup is held. + // The plugin-barman-cloud sidecar reads the ObjectStore through its own + // cache, kept for DefaultTTLSeconds = 10 (plugin-barman-cloud v0.13.0, + // internal/cnpgi/instance/internal/client/client.go) and invalidated + // only by the sidecar's own writes, so a Backup taken sooner can use the + // previous configuration and land in the previous bucket. The margin + // covers the cache's whole-second expiry and the record's whole-second + // timestamp. + cnpgObjectStoreCacheSettle = 15 * time.Second + + // How far ahead of this controller's clock a recorded time, of a plugin + // attach or of a destination change, may be before it is taken as + // corrupt rather than as written by a leader whose clock ran ahead. + cnpgAttachClockSkewTolerance = 5 * time.Minute + cnpgPodRoleLabel = "cnpg.io/podRole" cnpgPodRoleInstance = "instance" ) @@ -323,7 +386,53 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) awaitingRollout := existingBackup == nil && rollout != nil && rollout.Status == metav1.ConditionFalse - attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout) + // The ObjectStore is the Cluster's WAL archive destination, shared by + // every backup of the app, so it is written only before this job's + // Backup exists, and only while no other job's Backup is in flight and + // no other running job without a Backup yet holds the destination it + // last wrote. + writeDestination := existingBackup == nil + if writeDestination { + // The gate in Reconcile sees only a job that has not started, and + // this one may have waited in the queue for as long as a restore + // takes to begin. Read from the apiserver: the cache can still miss + // a RestoreJob created a moment ago. + restore, err := activeRestoreTargeting(ctx, r.apiReader(), j.Namespace, j.Spec.ApplicationRef) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld(ConditionReasonRestoreInProgress)) + return r.holdForRestore(ctx, j, restore) + } + holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName) + if err != nil { + return ctrl.Result{}, err + } + if holder == "" { + if holder, err = r.destinationClaimHolder(ctx, j, clusterName); err != nil { + return ctrl.Result{}, err + } + } + if holder != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForOtherBackup")) + return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf( + "waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder)) + } + if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil { + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionFalse, + Reason: "Released", + Message: "nothing holds the WAL archive destination write any more", + }) + } + } + + attemptedAt := metav1.Now() + attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination, j.Name) if err != nil { if apierrors.IsNotFound(err) { // HelmRelease has not yet rendered the Cluster (fresh app, or @@ -342,7 +451,29 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err)) + if !writeDestination { + // Only the live Cluster was read for a Backup already running; + // a failed read says nothing about that Backup. + return ctrl.Result{}, fmt.Errorf("read cnpg.io/Cluster %s/%s: %w", j.Namespace, clusterName, err) + } + message := fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err) + if apierrors.IsInvalid(err) || apierrors.IsBadRequest(err) || apierrors.IsForbidden(err) { + return r.markBackupJobFailed(ctx, j, message) + } + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondDestinationApplied, + Status: metav1.ConditionFalse, + Reason: "ClusterPluginApplyRetrying", + Message: message, + }) + failingSince := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied).LastTransitionTime + if cnpgBackupDeadlineExceeded(&failingSince) { + return r.markBackupJobFailed(ctx, j, message) + } + return r.waitCNPGBackupJob(ctx, j, "ClusterPluginApplyRetrying", message) + } + if writeDestination { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) } if attachment.serverName != serverName { logger.Info("preserving the live Cluster's barman serverName over the strategy template's", @@ -360,7 +491,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, - Reason: "WaitingForPluginRollout", + Reason: cnpgReasonAttachingPlugin, Message: message, }) apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ @@ -374,12 +505,54 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, rollout.LastTransitionTime) + if attachment.missing { + // This write attached the plugin, so its rollout starts here: + // instances recreated since the latch, or by a restore that + // re-rendered the Cluster without the plugin, run without it. A + // plugin someone else attached since the latch keeps it: the + // rollout that attach started may already be over. + message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName) + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondPluginRolledOut) + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + Message: message, + LastTransitionTime: attemptedAt, + }) + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + Message: message, + }) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil + } + // The Cluster's record covers an attach whose status write was lost. + attachedAt := rollout.LastTransitionTime + recorded, ok, err := cnpgRecordedTime(attachment.attachedAt) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf( + "cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v", + cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err)) + } + if ok && recorded.After(attachedAt.Time) { + attachedAt = metav1.NewTime(recorded) + } + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt) if err != nil { return ctrl.Result{}, err } if pending != "" { - if cnpgBackupDeadlineExceeded(j.Status.StartedAt) { + deadlineFrom := attachedAt + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil && + held.LastTransitionTime.After(deadlineFrom.Time) { + deadlineFrom = held.LastTransitionTime + } + if cnpgBackupDeadlineExceeded(&deadlineFrom) { return r.markBackupJobFailed(ctx, j, fmt.Sprintf( "barman-cloud plugin rollout did not finish within %s: %s", cnpgDefaultBackupDeadline, pending)) } @@ -403,6 +576,24 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp }) apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready") rolledOut = true + } else if existingBackup == nil && attachment.attachedAt != "" { + // Another backup job attached the plugin, and its rollout can still + // be running while the attach is recent; past the rollout deadline + // this job stops waiting. The record is that job's: one this job + // cannot use does not fail it. + recorded, _, err := cnpgRecordedTime(attachment.attachedAt) + if err != nil { + logger.Info("ignoring an unusable attach record another backup job left", + "cluster", clusterName, "annotation", cnpgPluginAttachedAtAnnotation, "error", err) + } else if time.Since(recorded) < cnpgDefaultBackupDeadline { + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, metav1.NewTime(recorded)) + if err != nil { + return ctrl.Result{}, err + } + if pending != "" { + return r.waitCNPGBackupJob(ctx, j, "WaitingForPluginRollout", pending) + } + } } // The chart refuses a flavor change on a live Cluster, but the app keeps @@ -413,7 +604,15 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp "cluster", clusterName, "live", attachment.flavor, "application", specFlavor) } - cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName) + if changedAt := attachment.destinationChangedAt; !changedAt.IsZero() && time.Since(changedAt) < cnpgObjectStoreCacheSettle { + return r.waitCNPGBackupJob(ctx, j, "WaitingForDestinationToSettle", fmt.Sprintf( + "the WAL archive destination of cnpg.io/Cluster %s/%s changed at %s; the barman-cloud plugin can read the previous one for up to %s", + j.Namespace, clusterName, changedAt.UTC().Format(time.RFC3339), cnpgObjectStoreCacheSettle)) + } + + written := cnpgDestinationOf(rendered) + written.ServerName = serverName + cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, existingBackup, written) if err != nil { return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure cnpg.io/Backup: %v", err)) } @@ -433,7 +632,16 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app) + destination, err := cnpgArtifactDestination(cnpgBackup, rendered) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("refusing to create the Backup artifact: %v", err)) + } + // A restore can re-render the Cluster under another serverName + // between the backup and this pass. + if destination.ServerName != "" { + serverName = destination.ServerName + } + artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, destination, rendered, app) if err != nil { return r.failBackupArtifact(ctx, j, err) } @@ -453,49 +661,50 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp return ctrl.Result{}, nil case cnpgBackupPhaseFailed: - // CNPG can transition the same Backup CR back out of "failed" when it - // retries internally (e.g., after a transient instance-manager - // restart), so we don't fail the BackupJob on the first observation. - // But without a cap a permanently-broken Backup would pin the - // BackupJob in Running forever; once StartedAt + cnpgDefaultBackupDeadline - // has elapsed and the cnpg.io/Backup is still failed, we give up. - if cnpgBackupDeadlineExceeded(j.Status.StartedAt) { - final := message - if final == "" { - final = fmt.Sprintf( - "cnpg.io Backup remained in phase=failed past %s deadline", - cnpgDefaultBackupDeadline) - } - return r.markBackupJobFailed(ctx, j, final) - } - if message == "" { - message = "cnpg.io Backup reported phase=failed; awaiting recovery or retry" - } - apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ - Type: "Ready", - Status: metav1.ConditionFalse, - Reason: "CNPGBackupTransientFailure", - Message: message, - }) - if err := r.Status().Update(ctx, j); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil + // CloudNativePG does not reconcile a Backup out of a terminal phase, + // so waiting on a failed one cannot change the outcome. + return r.markBackupJobFailed(ctx, j, cnpgBackupFailedMessage(cnpgBackup, message)) default: + waiting := fmt.Sprintf("waiting for cnpg.io/Backup %s/%s, phase %q", + cnpgBackup.Namespace, cnpgBackup.Name, phase) + if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "WaitingForCNPGBackup", + Message: waiting, + }) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } } -// cnpgBackupDeadlineExceeded reports whether enough wall-clock time has -// elapsed since the BackupJob started that we should stop polling a stuck -// cnpg.io/Backup and fail the run. Returns false when StartedAt is nil so -// the very first reconcile (which sets StartedAt) does not trip the gate. -func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool { - if startedAt == nil { +// cnpgBackupFailedMessage carries the error CloudNativePG recorded. When the +// plugin run itself failed that is only its exit status, and the storage +// error behind it is logged by the plugin-barman-cloud sidecar. +func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string { + if errText == "" { + errText = "no error recorded" + } + message := fmt.Sprintf("cnpg.io/Backup %s/%s failed: %s", b.Namespace, b.Name, errText) + if strings.Contains(errText, "exit status") { + message += "; the cause is in the log of the plugin-barman-cloud sidecar of the instance that ran it" + } + return message +} + +// cnpgBackupDeadlineExceeded reports whether more than the backup deadline +// has passed since the BackupJob began the step it is waiting on: the plugin +// rollout or retrying the destination write (see cnpgDefaultBackupDeadline). +// Returns false for a nil time. +func cnpgBackupDeadlineExceeded(since *metav1.Time) bool { + if since == nil { return false } - return time.Since(startedAt.Time) > cnpgDefaultBackupDeadline + return time.Since(since.Time) > cnpgDefaultBackupDeadline } // cnpgPurgeNeeded decides whether the controller should delete the live @@ -601,7 +810,10 @@ var cnpgClusterGVR = schema.GroupVersionResource{ // With attach unset, a Cluster that has no barman-cloud plugin yet is left // untouched and only reported missing, so the caller can record that it is // about to roll the Cluster before it does. -func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach bool) (cnpgPluginAttachment, error) { +// +// With write unset nothing is written: only what the live Cluster carries is +// reported. +func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach, write bool, jobName string) (cnpgPluginAttachment, error) { // Live read: the cached Cluster can predate a restore re-render, and the // serverName, the flavor and the UID below must come from the current // object. @@ -618,6 +830,7 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name flavor: cnpgImageFlavor(existing.Spec.ImageName), clusterStatus: existing.Status, } + attachment.attachedAt = existing.Annotations[cnpgPluginAttachedAtAnnotation] if live := currentBarmanServerName(existing); live != "" { attachment.serverName = live } else { @@ -626,6 +839,9 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name return attachment, nil } } + if !write { + return attachment, nil + } serverName = attachment.serverName // The ObjectStore is named after the Cluster (distinct kind, same @@ -653,18 +869,116 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, InstanceSidecarConfiguration: barmanSidecarConfiguration(), } + changedAt, err := r.objectStoreChangeRecord(ctx, objStore) + if err != nil { + return cnpgPluginAttachment{}, err + } + // Only a job without a Backup of its own writes, and the queue let it + // through, so it holds the destination from this write while it runs + // and has not created its Backup. + objStore.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: jobName} + if !changedAt.IsZero() { + objStore.Annotations[cnpgDestinationChangedAtAnnotation] = changedAt.UTC().Format(time.RFC3339) + } if err := r.Patch(ctx, objStore, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return cnpgPluginAttachment{}, fmt.Errorf("apply ObjectStore %s/%s: %w", namespace, objStoreName, err) } + attachment.destinationChangedAt = changedAt patch := newCNPGClusterPatch(namespace, clusterName) patch.Spec.Plugins = []cnpgtypes.PluginConfiguration{buildBarmanPlugin(objStoreName, serverName)} + // Re-sent unchanged on later applies: the field manager owns it, and an + // apply that left it out would delete it. + stamp := existing.Annotations[cnpgPluginAttachedAtAnnotation] + if attachment.missing { + stamp = metav1.Now().UTC().Format(time.RFC3339) + } + if stamp != "" { + patch.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + } if err := r.Patch(ctx, patch, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return cnpgPluginAttachment{}, err } return attachment, nil } +// objectStoreChangeRecord returns the cnpgDestinationChangedAtAnnotation the +// apply of desired should carry: the time of this write when it changes the +// configuration, otherwise the record the live ObjectStore already carries, +// re-sent so the apply does not delete it, and zero when there is none. +// "Changes" is judged on the configuration a server-side dry run of the same +// apply returns, not on desired, so values the apiserver defaults or another +// field manager owns do not read as a change on every pass. Creating the +// ObjectStore stamps no time: the plugin cannot hold a stale copy of an +// object that did not exist, short of one deleted and recreated by hand +// within the cache's lifetime. +func (r *BackupJobReconciler) objectStoreChangeRecord(ctx context.Context, desired *cnpgtypes.ObjectStore) (time.Time, error) { + live := &cnpgtypes.ObjectStore{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil { + if apierrors.IsNotFound(err) { + return time.Time{}, nil + } + return time.Time{}, fmt.Errorf("read ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err) + } + dry := desired.DeepCopy() + if err := r.Patch(ctx, dry, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership, client.DryRunAll); err != nil { + return time.Time{}, fmt.Errorf("dry-run apply ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err) + } + if !apiequality.Semantic.DeepEqual(live.Spec.Configuration, dry.Spec.Configuration) { + return time.Now(), nil + } + recorded, _, err := cnpgRecordedTime(live.Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil { + // Nothing says when the cache expires; this write's time is the + // latest it can be. + return time.Now(), nil + } + return recorded, nil +} + +// destinationClaimHolder returns the BackupJob other than j that the +// ObjectStore of clusterName names as the last to write it, while that job +// is still running and has no cnpg.io/Backup of its own, or "" when there is +// none. Such a job wrote the destination before otherCNPGBackupInFlight can +// see it: it can still be waiting for the plugin cache, or for the rollout of +// the plugin it attached. Once it has a Backup, that Backup holds the +// destination while CloudNativePG runs it, and the claim stops holding, so a +// job stuck after its Backup finished does not hold every later one. +func (r *BackupJobReconciler) destinationClaimHolder(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + store := &cnpgtypes.ObjectStore{} + if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: clusterName}, store); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", fmt.Errorf("read ObjectStore %s/%s: %w", j.Namespace, clusterName, err) + } + holder := store.Annotations[cnpgDestinationWrittenByAnnotation] + if holder == "" || holder == j.Name { + return "", nil + } + holderJob := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: holder}, holderJob); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", err + } + if holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + return "", nil + } + holderBackups := &cnpgtypes.BackupList{} + if err := r.apiReader().List(ctx, holderBackups, client.InNamespace(j.Namespace), client.MatchingLabels{ + backupsv1alpha1.OwningJobNameLabel: holder, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }); err != nil { + return "", fmt.Errorf("list cnpg.io/Backups of BackupJob %s/%s: %w", j.Namespace, holder, err) + } + if len(holderBackups.Items) > 0 { + return "", nil + } + return holder, nil +} + // cnpgPluginAttachment is what applyClusterPluginBackup read off the live // Cluster before applying the plugin. type cnpgPluginAttachment struct { @@ -673,8 +987,15 @@ type cnpgPluginAttachment struct { // flavor is the image flavor the live Cluster runs. flavor string // missing reports a Cluster that carried no barman-cloud plugin. - missing bool - clusterStatus cnpgtypes.ClusterStatus + missing bool + // attachedAt is the cnpgPluginAttachedAtAnnotation the Cluster carried, + // "" when it carries no record: the plugin was attached by the chart, or + // before the record existed. + attachedAt string + // destinationChangedAt is the cnpgDestinationChangedAtAnnotation this + // pass applied to the ObjectStore, zero when it wrote none. + destinationChangedAt time.Time + clusterStatus cnpgtypes.ClusterStatus } // cnpgPluginRolloutPending returns why the rollout that attaching the @@ -720,6 +1041,26 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name return "", nil } +// cnpgRecordedTime parses a recorded time annotation, the plugin attach +// record on the Cluster or the destination change record on the ObjectStore; +// ok is false for "", an object that carries no record. A record further +// ahead of this controller's clock than cnpgAttachClockSkewTolerance is an +// error: every instance would predate it, and a deadline counted from it +// would not run out for as long as it lies ahead. +func cnpgRecordedTime(raw string) (recorded time.Time, ok bool, err error) { + if raw == "" { + return time.Time{}, false, nil + } + recorded, err = time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, false, err + } + if time.Until(recorded) > cnpgAttachClockSkewTolerance { + return time.Time{}, false, fmt.Errorf("%s is in the future", raw) + } + return recorded, true, nil +} + func podReady(pod *corev1.Pod) bool { for _, c := range pod.Status.Conditions { if c.Type == corev1.PodReady { @@ -747,16 +1088,23 @@ func currentBarmanServerName(c *cnpgtypes.Cluster) string { } // ensureCNPGBackup creates a one-shot postgresql.cnpg.io/Backup CR labelled -// with the BackupJob, or returns the existing one if a previous reconcile -// already created it. Idempotency relies on the OwningJob labels. -func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (*cnpgtypes.Backup, error) { - existing, err := r.findCNPGBackupForJob(ctx, j) - if err != nil { - return nil, err - } +// with the BackupJob, or returns existing, the job's Backup as looked up at +// the start of the pass. It does not look again: a reconcile of the same +// job cannot run in between, and a failed lookup here would come after the +// destination write and fail the job. +// +// written is the destination this pass wrote to the ObjectStore. A pass +// reaches the create only after that write: a job without a Backup writes +// the destination on every pass, and a pass that wrote nothing, or wrote it +// and waits for the plugin rollout, returns before this point. +func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, existing *cnpgtypes.Backup, written cnpgDestination) (*cnpgtypes.Backup, error) { if existing != nil { return existing, nil } + record, err := json.Marshal(written) + if err != nil { + return nil, fmt.Errorf("encode the WAL archive destination: %w", err) + } obj := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -767,6 +1115,7 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1 backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, cnpgClusterLabel: clusterName, }, + Annotations: map[string]string{cnpgWrittenDestinationAnnotation: string(record)}, }, Spec: cnpgtypes.BackupSpec{ Method: cnpgtypes.BackupMethodPlugin, @@ -784,23 +1133,167 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1 // findCNPGBackupForJob returns the cnpg.io/Backup labelled with the // BackupJob's OwningJob{Name,Namespace}, if any. Returns (nil, nil) when // no match is found. +// +// A miss in the cache is checked against the apiserver: the cache can lag +// behind a Backup the previous pass created, and taking that miss as no +// Backup would write the destination under it and create a second one. func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backupsv1alpha1.BackupJob) (*cnpgtypes.Backup, error) { - list := &cnpgtypes.BackupList{} - if err := r.List(ctx, list, + opts := []client.ListOption{ client.InNamespace(j.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: j.Name, backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, }, - ); err != nil { + } + list := &cnpgtypes.BackupList{} + if err := r.List(ctx, list, opts...); err != nil { return nil, err } + if len(list.Items) == 0 { + if err := r.apiReader().List(ctx, list, opts...); err != nil { + return nil, err + } + } if len(list.Items) == 0 { return nil, nil } return &list.Items[0], nil } +// otherCNPGBackupInFlight returns the name of another BackupJob that is still +// running and whose cnpg.io/Backup of clusterName has not finished, or "" when +// there is none. A Backup left behind by a finished or deleted job does not +// hold the queue. +// +// The Backups are read from the apiserver, not the cache, which can still +// miss a Backup another job's earlier pass created. Only the ones this +// controller created for the Cluster are asked for, by the labels +// ensureCNPGBackup has always set, so a waiting job's poll does not fetch +// the namespace's whole Backup history. +func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + list := &cnpgtypes.BackupList{} + if err := r.apiReader().List(ctx, list, client.InNamespace(j.Namespace), + client.MatchingLabels{cnpgClusterLabel: clusterName}, + client.HasLabels{backupsv1alpha1.OwningJobNameLabel}); err != nil { + return "", err + } + var holders []string + for i := range list.Items { + b := &list.Items[i] + owner := b.Labels[backupsv1alpha1.OwningJobNameLabel] + if b.Spec.Cluster.Name != clusterName || owner == "" || owner == j.Name { + continue + } + if b.Status.Phase == cnpgBackupPhaseComplete || b.Status.Phase == cnpgBackupPhaseFailed { + continue + } + ownerJob := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: owner}, ownerJob); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return "", err + } + if ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + continue + } + holders = append(holders, owner) + } + if len(holders) == 0 { + return "", nil + } + sort.Strings(holders) + return holders[0], nil +} + +func cnpgDestinationWriteHeld(reason string) metav1.Condition { + return metav1.Condition{ + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionTrue, + Reason: reason, + Message: "held off writing the WAL archive destination", + } +} + +// waitCNPGBackupJob requeues the BackupJob and records why it waits in its +// Ready condition, writing the status only when the conditions changed. +func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv1alpha1.BackupJob, reason, message string) (ctrl.Result, error) { + if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + }) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil +} + +// cnpgDestination is the WAL archive destination recorded in +// cnpgWrittenDestinationAnnotation. S3SecretRef is nil when the strategy +// names no credentials Secret. ServerName is empty in a record written +// before it carried one. +type cnpgDestination struct { + DestinationPath string `json:"destinationPath"` + EndpointURL string `json:"endpointURL,omitempty"` + S3SecretRef *string `json:"s3SecretRef,omitempty"` + ServerName string `json:"serverName,omitempty"` +} + +func cnpgDestinationOf(t *strategyv1alpha1.CNPGTemplate) cnpgDestination { + d := cnpgDestination{ + DestinationPath: t.BarmanObjectStore.DestinationPath, + EndpointURL: t.BarmanObjectStore.EndpointURL, + } + if t.BarmanObjectStore.S3Credentials != nil { + name := t.BarmanObjectStore.S3Credentials.SecretRef.Name + d.S3SecretRef = &name + } + return d +} + +// cnpgArtifactDestination returns the destination the artifact of b names: +// the one recorded when b was created. A Backup without the record was +// created by a controller that kept none, and falls back to the strategy +// rendered now. A record that names no destination is an error: the +// strategy may have changed since the write, so it cannot stand in. +func cnpgArtifactDestination(b *cnpgtypes.Backup, rendered *strategyv1alpha1.CNPGTemplate) (cnpgDestination, error) { + raw, ok := b.Annotations[cnpgWrittenDestinationAnnotation] + if !ok { + return cnpgDestinationOf(rendered), nil + } + var d cnpgDestination + if err := json.Unmarshal([]byte(raw), &d); err != nil { + return cnpgDestination{}, fmt.Errorf("read the %s annotation of cnpg.io/Backup %s/%s: %w", cnpgWrittenDestinationAnnotation, b.Namespace, b.Name, err) + } + if d.DestinationPath == "" { + return cnpgDestination{}, fmt.Errorf("the %s annotation of cnpg.io/Backup %s/%s names no destination path", cnpgWrittenDestinationAnnotation, b.Namespace, b.Name) + } + return d, nil +} + +// cnpgProbeStore is the object store the size probe reads: the location +// and the credentials Secret recorded for the backup, with the Secret keys +// and the endpoint CA the record does not carry taken from the strategy. +// After a strategy edit those may not fit the recorded location; the probe +// then reports no size rather than measuring somewhere else. +func cnpgProbeStore(rendered *strategyv1alpha1.BarmanObjectStoreTemplate, destination cnpgDestination) *strategyv1alpha1.BarmanObjectStoreTemplate { + store := rendered.DeepCopy() + store.DestinationPath = destination.DestinationPath + store.EndpointURL = destination.EndpointURL + if destination.S3SecretRef == nil { + store.S3Credentials = nil + } else { + if store.S3Credentials == nil { + store.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{} + } + store.S3Credentials.SecretRef.Name = *destination.S3SecretRef + } + return store +} + // createCNPGBackupArtifact materialises a Cozystack Backup resource carrying // the metadata callers need to drive a future restore. The source app's // spec.databases / spec.users are snapshotted into Status.UnderlyingResources @@ -812,6 +1305,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( resolved *ResolvedBackupConfig, cnpgBackup *cnpgtypes.Backup, clusterName, serverName, flavor string, + destination cnpgDestination, rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { @@ -827,12 +1321,12 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( driverMD := map[string]string{ cnpgBackupNameKey: cnpgBackup.Name, cnpgServerNameKey: serverName, - cnpgDestinationPathKey: rendered.BarmanObjectStore.DestinationPath, - cnpgEndpointURLKey: rendered.BarmanObjectStore.EndpointURL, + cnpgDestinationPathKey: destination.DestinationPath, + cnpgEndpointURLKey: destination.EndpointURL, cnpgClusterNameKey: clusterName, } - if rendered.BarmanObjectStore.S3Credentials != nil { - driverMD[cnpgS3SecretRefKey] = rendered.BarmanObjectStore.S3Credentials.SecretRef.Name + if destination.S3SecretRef != nil { + driverMD[cnpgS3SecretRefKey] = *destination.S3SecretRef } // Copied rather than read back at restore time: barman retention deletes // the cnpg.io/Backup while the artifact can still be restored from. @@ -865,7 +1359,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( }, } if probe { - backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID) + backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, cnpgProbeStore(&rendered.BarmanObjectStore, destination), serverName, cnpgBackup.Status.BackupID) } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 697843d9e..5cd990f18 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -3,7 +3,9 @@ package backupcontroller import ( "context" + "errors" "fmt" + "slices" "strings" "testing" "time" @@ -13,13 +15,16 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" + clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -997,10 +1002,9 @@ func TestUnmarshalCNPGBackupSnapshot_RejectsMissingOrWrongKind(t *testing.T) { } } -// TestCNPGBackupDeadlineExceeded locks in the wall-clock cap that prevents a -// perpetually-failed cnpg.io/Backup from pinning the BackupJob in Running -// forever. Without this guard the Plan-controller queue would back up -// behind a single broken backup. +// TestCNPGBackupDeadlineExceeded locks in the wall-clock cap on waiting for +// the barman-cloud plugin rollout, so a rollout that never finishes fails the +// BackupJob instead of pinning it in Running. func TestCNPGBackupDeadlineExceeded(t *testing.T) { cases := []struct { name string @@ -1073,7 +1077,7 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsReturnsExisting(t *testing.T) { } sourceApp := newPostgresApp("pg", "tenant") - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", rendered, sourceApp) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, sourceApp) if err != nil { t.Fatalf("expected AlreadyExists to be swallowed, got error %v", err) } @@ -1099,7 +1103,7 @@ func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) { }, } - _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true) + _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true, "bj") if err == nil { t.Fatalf("expected NotFound error, got nil") } @@ -1739,7 +1743,7 @@ func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) { BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } - got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true) + got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1949,7 +1953,7 @@ func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) { BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } - got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true) + got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2118,7 +2122,7 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { }, } - if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true); err != nil { + if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true, "bj"); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2205,7 +2209,7 @@ func TestEnsureCNPGBackup_UsesPluginMethod(t *testing.T) { c := newCNPGStrategyTestClient(t, j) r := &BackupJobReconciler{Client: c} - bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app") + bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", nil, cnpgDestination{DestinationPath: "s3://bucket/"}) if err != nil { t.Fatalf("ensureCNPGBackup: %v", err) } @@ -2332,6 +2336,1855 @@ func TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion(t *testi } } +// CloudNativePG never reconciles a Backup out of a terminal phase (the +// backup controller returns early on failed and completed), so a failed +// Backup fails the BackupJob at once. Any other phase keeps the BackupJob +// waiting past the backup deadline: CNPG keeps a healthy Backup pending +// while another backup of the Cluster runs or while it restarts one. +func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + const ns, appName = "tenant", "pg" + clusterName := cnpgClusterNameForApp(appName) + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + + cases := []struct { + name string + phase string + errText string + startedAt time.Time + wantPhase backupsv1alpha1.BackupJobPhase + wantMessage []string + notInMessage []string + }{ + {name: "failed fails at once", phase: cnpgBackupPhaseFailed, errText: "rpc error: code = Unknown desc = exit status 1", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "exit status 1", "plugin-barman-cloud"}}, + {name: "failed without an error text fails at once", phase: cnpgBackupPhaseFailed, startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "no error recorded"}, notInMessage: []string{"plugin-barman-cloud"}}, + {name: "failed before the plugin ran points nowhere else", phase: cnpgBackupPhaseFailed, errText: "cannot backup a hibernated cluster", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "hibernated"}, notInMessage: []string{"plugin-barman-cloud"}}, + {name: "pending keeps waiting", phase: "pending", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, + {name: "pending is not capped by the backup deadline", phase: "pending", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, + {name: "no phase yet keeps waiting", phase: "", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg"}}, + {name: "started keeps running", phase: "started", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "started"}}, + {name: "walArchivingFailing keeps waiting", phase: "walArchivingFailing", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "walArchivingFailing"}}, + {name: "completed succeeds", phase: cnpgBackupPhaseComplete, startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseSucceeded}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + startedAt := metav1.NewTime(tc.startedAt) + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, + Spec: strategyv1alpha1.CNPGSpec{ + Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, + }, + }, + } + cluster := &cnpgtypes.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"}, + } + cnpgBackup := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: job.Name, + backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace, + }}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, + Status: cnpgtypes.BackupStatus{Phase: tc.phase, Error: tc.errText}, + } + c := newCNPGStrategyTestClient(t, job, strategy, newPostgresApp(appName, ns), cluster, cnpgBackup) + r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}, + } + + j := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, err := r.reconcileCNPG(context.Background(), j, resolved) + if err != nil { + t.Fatalf("reconcileCNPG: %v", err) + } + if j.Status.Phase != tc.wantPhase { + t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) + } + stored := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), stored); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + if stored.Status.Phase != tc.wantPhase { + t.Fatalf("stored BackupJob phase = %q, want %q", stored.Status.Phase, tc.wantPhase) + } + + message := j.Status.Message + if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning { + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the cnpg.io/Backup is in progress, got %+v", res) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), persisted); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + ready := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForCNPGBackup" { + t.Fatalf("expected Ready=False/WaitingForCNPGBackup while waiting, got %+v", ready) + } + message = ready.Message + + // A pass that observes the same phase again must not write the + // BackupJob status: the driver polls every few seconds. + statusWrites := 0 + r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + statusWrites++ + return cl.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) + if _, err := r.reconcileCNPG(context.Background(), persisted, resolved); err != nil { + t.Fatalf("second reconcileCNPG: %v", err) + } + if statusWrites != 0 { + t.Errorf("an unchanged phase wrote the BackupJob status %d time(s)", statusWrites) + } + } + for _, want := range tc.wantMessage { + if !strings.Contains(message, want) { + t.Errorf("message %q does not mention %q", message, want) + } + } + for _, unwanted := range tc.notInMessage { + if strings.Contains(message, unwanted) { + t.Errorf("message %q mentions %q", message, unwanted) + } + } + + artifacts := &backupsv1alpha1.BackupList{} + if err := c.List(context.Background(), artifacts, client.InNamespace(ns)); err != nil { + t.Fatalf("list Backups: %v", err) + } + wantArtifacts := 0 + if tc.wantPhase == backupsv1alpha1.BackupJobPhaseSucceeded { + wantArtifacts = 1 + } + if len(artifacts.Items) != wantArtifacts { + t.Errorf("got %d Backup artifacts, want %d", len(artifacts.Items), wantArtifacts) + } + }) + } +} + +// newAttachedObjectStoreFixture is a Cluster that already runs the barman-cloud +// plugin, whose ObjectStore currently points at liveDestination; the strategy +// of the fixture's BackupJob points at s3://bucket/. +func newAttachedObjectStoreFixture(t *testing.T, startedAt time.Time, liveDestination string) *cnpgRolloutFixture { + t.Helper() + f := newCNPGRolloutFixture(t, newRolloutBackupJob(startedAt), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + store := &cnpgtypes.ObjectStore{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster}, + Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: liveDestination}}, + } + if err := f.c.Create(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + return f +} + +func (f *cnpgRolloutFixture) liveDestination() string { + f.t.Helper() + store := &cnpgtypes.ObjectStore{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { + f.t.Fatalf("get ObjectStore: %v", err) + } + return store.Spec.Configuration.DestinationPath +} + +// seedCNPGBackup creates a cnpg.io/Backup of cluster owned by the BackupJob +// named owner, labelled as ensureCNPGBackup labels it; an empty owner leaves +// it unlabelled, as a Backup created by hand. +func (f *cnpgRolloutFixture) seedCNPGBackup(name, owner, cluster, phase string) { + f.t.Helper() + var backupLabels map[string]string + if owner != "" { + backupLabels = map[string]string{ + backupsv1alpha1.OwningJobNameLabel: owner, + backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, + cnpgClusterLabel: cluster, + } + } + b := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: name, Labels: backupLabels}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cluster}}, + Status: cnpgtypes.BackupStatus{Phase: phase}, + } + if err := f.c.Create(context.Background(), b); err != nil { + f.t.Fatalf("seed cnpg.io/Backup %s: %v", name, err) + } +} + +func (f *cnpgRolloutFixture) setJobCondition(cond metav1.Condition) { + f.t.Helper() + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + f.t.Fatalf("get BackupJob: %v", err) + } + j.Status.Conditions = append(j.Status.Conditions, cond) + if err := f.c.Status().Update(context.Background(), j); err != nil { + f.t.Fatalf("seed BackupJob condition: %v", err) + } +} + +func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alpha1.BackupJobPhase) { + f.t.Helper() + other := newRolloutBackupJob(time.Now().Add(-time.Hour)) + other.Name = name + other.Status.Phase = phase + if err := f.c.Create(context.Background(), other); err != nil { + f.t.Fatalf("seed BackupJob %s: %v", name, err) + } +} + +// The ObjectStore is the Cluster's WAL archive destination, so a BackupJob +// whose cnpg.io/Backup is already running must not keep re-applying it: a +// second BackupJob of the same app with another destination would otherwise +// flip it back and forth under both backups. +func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + // Two jobs can still pass the queue together, across a leader handover; + // once both have a Backup neither may wait on the other. + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + writes := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + writes++ + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + j, _ := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("a pass over a running backup re-applied the ObjectStore: destination %q", got) + } + if writes != 0 { + t.Errorf("a pass over a running backup patched %d object(s); the ObjectStore and the Cluster plugin are only read", writes) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Fatalf("Ready = %+v, want the running backup followed rather than queued", ready) + } +} + +// The queue reads the apiserver on every pass of every waiting job, so it +// asks only for the Backups this controller created for the Cluster rather +// than the namespace's whole Backup history. +func TestReconcileCNPG_QueueListsOnlyTheClustersOwnedBackups(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + var selectors []labels.Selector + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cnpgtypes.BackupList); ok { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + selectors = append(selectors, lo.LabelSelector) + } + return cl.List(ctx, list, opts...) + }, + }) + f.pass() + owned := labels.Set{cnpgClusterLabel: rolloutCluster, backupsv1alpha1.OwningJobNameLabel: "bj-other"} + foreignCluster := labels.Set{cnpgClusterLabel: "postgres-other", backupsv1alpha1.OwningJobNameLabel: "bj-other"} + unowned := labels.Set{cnpgClusterLabel: rolloutCluster} + var queue labels.Selector + for _, s := range selectors { + if s == nil { + s = labels.Everything() + } + if s.Matches(owned) { + queue = s + } + } + if queue == nil { + t.Fatalf("no cnpg.io/Backup list would find another job's Backup; selectors = %v", selectors) + } + if queue.Matches(foreignCluster) || queue.Matches(unowned) { + t.Errorf("queue selector %q, want it to match only owned Backups of %s", queue, rolloutCluster) + } +} + +func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { + cases := []struct { + name string + otherJob backupsv1alpha1.BackupJobPhase + seedJob bool + otherBackup string + owner string + cluster string + wantWait bool + }{ + {name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true}, + {name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true}, + {name: "a completed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseComplete}, + {name: "a failed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseFailed}, + {name: "a Backup of a failed job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "pending"}, + {name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"}, + {name: "a Backup whose job is gone does not", otherBackup: "pending"}, + {name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"}, + {name: "a Backup created by hand does not", otherBackup: "running", owner: "-"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + if tc.seedJob { + f.seedOtherBackupJob("bj-other", tc.otherJob) + } + owner, cluster := "bj-other", rolloutCluster + if tc.owner == "-" { + owner = "" + } + if tc.cluster != "" { + cluster = tc.cluster + } + f.seedCNPGBackup("bj-other-cnpg", owner, cluster, tc.otherBackup) + // client-go refuses a Get with an empty name before it reaches + // the apiserver; the fake client answers NotFound instead. + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return cl.Get(ctx, key, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + ours := 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + ours++ + } + } + ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") + if tc.wantWait { + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("the waiting job applied its destination over a running backup: %q", got) + } + if ours != 0 { + t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours) + } + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { + t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res) + } + statusWrites = 0 + if _, _, res = f.passResult(); res.RequeueAfter == 0 { + t.Errorf("a queued job whose status did not change is not requeued: %+v", res) + } + if statusWrites != 0 { + t.Errorf("a second pass with the same holder wrote the BackupJob status %d time(s)", statusWrites) + } + + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + j, _ = f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready still names the finished holder: %+v", ready) + } + return + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want the job's own s3://bucket/", got) + } + if ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, the job was held", ready) + } + f.settleDestination() + _, backups = f.pass() + ours = 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + ours++ + } + } + if ours != 1 { + t.Errorf("got %d cnpg.io/Backups for the job, want 1", ours) + } + }) + } +} + +func TestReconcileCNPG_ObjectStoreApplyErrors(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + webhookDown := apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io": connect: connection refused`)) + invalid := apierrors.NewInvalid(schema.GroupKind{Group: cnpgtypes.BarmanGroupVersion.Group, Kind: "ObjectStore"}, rolloutCluster, nil) + cases := []struct { + name string + err error + startedAt time.Time + // failingSince seeds an earlier failure of the same write. + failingSince time.Time + wantPhase backupsv1alpha1.BackupJobPhase + }{ + {name: "a transient error is retried", err: webhookDown, startedAt: time.Now()}, + {name: "a transient error failing since past the deadline fails", err: webhookDown, startedAt: expired, failingSince: expired, wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a transient error after a long wait is retried", err: webhookDown, startedAt: expired}, + {name: "an invalid object fails at once", err: invalid, startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a bad request fails at once", err: apierrors.NewBadRequest("malformed"), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a forbidden write fails at once", err: apierrors.NewForbidden(schema.GroupResource{Resource: "objectstores"}, rolloutCluster, errors.New("denied")), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, tc.startedAt, "s3://other/") + if !tc.failingSince.IsZero() { + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(tc.failingSince), + }) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return tc.err + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + j, backups, res := f.passResult() + if j.Status.Phase != tc.wantPhase { + t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created although the ObjectStore was not applied: %+v", backups) + } + if tc.wantPhase == "" { + ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") + if ready == nil || ready.Reason != "ClusterPluginApplyRetrying" || !strings.Contains(ready.Message, "connection refused") { + t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying carrying the error", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a retried write is not requeued: %+v", res) + } + } + }) + } +} + +// A write that recovers clears the retry state, so a later failure gets a +// fresh budget and Ready no longer reports the retry. +func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + failing := true + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && failing { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond == nil || cond.Status != metav1.ConditionFalse { + t.Fatalf("%s = %+v after a failed write, want False", backupCondDestinationApplied, cond) + } + + failing = false + j, _ = f.pass() + f.settleDestination() + j, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the write recovered, want 1", len(backups)) + } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Errorf("%s = %+v left over after the write recovered", backupCondDestinationApplied, cond) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "ClusterPluginApplyRetrying" { + t.Errorf("Ready still reports the retry: %+v", ready) + } +} + +// An error reading the holder's BackupJob is not taken as "no holder". +func TestReconcileCNPG_HolderLookupErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "bj-other" { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the holder lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the holder could not be read: %q", got) + } +} + +// The rollout deadline counts from when the plugin was attached, not from +// the job's start, which may lie behind a long wait for another backup. +func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + + f.pass() // latch + j, _ := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that just began failed on the job's old start: %s", j.Status.Message) + } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond) + } + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) + + rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + rollout.LastTransitionTime = metav1.NewTime(expired) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the rollout: %v", err) + } + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed on a rollout older than the deadline", j.Status.Phase, j.Status.Message) + } +} + +// A rollout dated by the Cluster's attach record runs out like one dated by +// the job's own condition: a record newer than the condition but older than +// the deadline fails the job too. +func TestReconcileCNPG_PluginRolloutDeadlineCountsFromTheAttachRecord(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(expired.Add(-time.Hour))) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(expired.Add(-10 * time.Minute)) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: expired.UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed on a rollout recorded longer ago than the deadline", j.Status.Phase, j.Status.Message) + } +} + +// The rollout is measured from the write that attached the plugin, not from +// the latch recorded before it: retries of a failing write can lie between +// the two, and an instance recreated in that time runs without the plugin. +func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) { + cases := []struct { + name string + // podBetween recreates the instance after the latch and before the + // attach; otherwise it predates both. + podBetween bool + }{ + {name: "an instance recreated before the attach still has to roll", podBetween: true}, + {name: "retries before the attach do not spend the rollout deadline"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + oldPod := newRolloutInstancePod(expired.Add(-time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(expired) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + + failing := true + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok && failing { + return apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io"`)) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a failed attach failed the job: %s", j.Status.Message) + } + if tc.podBetween { + f.replacePod(oldPod, newRolloutInstancePod(expired.Add(time.Minute))) + } + + failing = false + attachAt := time.Now().Truncate(time.Second) + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups) + } + rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { + t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) + } + + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) + j, backups = f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that began with the attach failed: %s", j.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || !strings.Contains(ready.Message, "predates") { + t.Errorf("Ready = %+v, want the instance that predates the plugin", ready) + } + }) + } +} + +// The status write that records the attach can be lost after the attach +// itself went through, by a conflict, an apiserver that is briefly +// unavailable, or a restart, including on the retry of an attach that +// failed; an instance recreated between the latch and the attach still has +// to roll, and the rollout deadline still counts from the attach. +func TestReconcileCNPG_LostAttachRecordStillWaitsForTheRollout(t *testing.T) { + jobs := backupsv1alpha1.GroupVersion.WithResource("backupjobs").GroupResource() + for _, tc := range []struct { + name string + failFirstAttach bool + statusWriteError error + }{ + {name: "conflict", statusWriteError: apierrors.NewConflict(jobs, "bj", errors.New("the object has been modified"))}, + {name: "retried attach, apiserver unavailable", failFirstAttach: true, statusWriteError: apierrors.NewServiceUnavailable("the server is currently unable to handle the request")}, + } { + t.Run(tc.name, func(t *testing.T) { + latchedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + oldPod := newRolloutInstancePod(latchedAt.Add(-time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(time.Minute))) + + base := f.r.Client + if tc.failFirstAttach { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok { + return apierrors.NewServiceUnavailable("the server is currently unable to handle the request") + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { + t.Fatalf("a failed attach reported the rollout done: %+v", j.Status.Conditions) + } + if applied := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); applied == nil || applied.Reason != "ClusterPluginApplyRetrying" { + t.Fatalf("%s = %+v, want the attach retrying", backupCondDestinationApplied, applied) + } + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return tc.statusWriteError + }, + }) + j = &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("the pass that attached the plugin reported no lost status write") + } + f.r.Client = base + + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + if len(attached.Spec.Plugins) == 0 { + t.Fatalf("the plugin was not attached") + } + live := newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...) + live.Annotations = attached.Annotations + f.observeCNPG(live) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the attach: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) + } + }) + } +} + +// A record older than the latch is left from an earlier attach of a plugin +// since removed; the plugin found now was attached by someone who keeps no +// record, after the latch, so an instance recreated between the record and +// the latch still has to roll. +func TestReconcileCNPG_AttachRecordOlderThanTheLatchKeepsTheLatch(t *testing.T) { + latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + oldPod := newRolloutInstancePod(latchedAt.Add(-2 * time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-2*time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(-time.Minute))) + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: latchedAt.Add(-time.Hour).UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the latch: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) + } +} + +func (f *cnpgRolloutFixture) liveObjectStore() *cnpgtypes.ObjectStore { + f.t.Helper() + store := &cnpgtypes.ObjectStore{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { + f.t.Fatalf("get ObjectStore: %v", err) + } + return store +} + +// setDestinationStamp rewrites the destination-change record on the live +// ObjectStore, as a write that long ago, or a corrupt one, would have left it. +func (f *cnpgRolloutFixture) setDestinationStamp(value string) { + f.t.Helper() + store := f.liveObjectStore() + if store.Annotations == nil { + store.Annotations = map[string]string{} + } + store.Annotations[cnpgDestinationChangedAtAnnotation] = value + if err := f.c.Update(context.Background(), store); err != nil { + f.t.Fatalf("stamp ObjectStore: %v", err) + } +} + +// settleDestination ages the destination-change record, when the ObjectStore +// carries one, as if the plugin's cache had expired since that write. +func (f *cnpgRolloutFixture) settleDestination() { + f.t.Helper() + if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { + f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) + } +} + +// addJobWithDestination adds BackupJob name of the same application, resolved +// to a strategy of its own that writes destination, and returns a function +// running one pass of it. +func (f *cnpgRolloutFixture) addJobWithDestination(name, destination string) func() *backupsv1alpha1.BackupJob { + f.t.Helper() + ctx := context.Background() + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy-" + name}, + Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: destination}, + }}, + } + if err := f.c.Create(ctx, strategy); err != nil { + f.t.Fatalf("seed strategy: %v", err) + } + job := newRolloutBackupJob(time.Now().Add(-time.Hour)) + job.Name = name + if err := f.c.Create(ctx, job); err != nil { + f.t.Fatalf("seed BackupJob %s: %v", name, err) + } + group := strategyv1alpha1.GroupVersion.Group + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &group, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}} + return func() *backupsv1alpha1.BackupJob { + f.t.Helper() + key := client.ObjectKey{Namespace: rolloutNS, Name: name} + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, key, j); err != nil { + f.t.Fatalf("get BackupJob %s: %v", name, err) + } + if _, err := f.r.reconcileCNPG(ctx, j, resolved); err != nil { + f.t.Fatalf("reconcileCNPG %s: %v", name, err) + } + if err := f.c.Get(ctx, key, j); err != nil { + f.t.Fatalf("get BackupJob %s after reconcile: %v", name, err) + } + return j + } +} + +// Two jobs of the application with different destinations, neither with a +// Backup yet, would otherwise take turns rewriting the ObjectStore, each +// write restarting the other's wait for the plugin cache so that neither +// ever creates one. The job whose write came first holds the destination, +// by its claim until it has a Backup and by that Backup afterwards, and the +// other waits for it. +func TestReconcileCNPG_JobsWithDifferentDestinationsTakeTurns(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://third/") + pass2 := f.addJobWithDestination("bj2", "s3://other/") + + var j2 *backupsv1alpha1.BackupJob + for range 5 { + f.pass() + f.settleDestination() + j2 = pass2() + f.settleDestination() + } + backups := &cnpgtypes.BackupList{} + if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { + t.Fatalf("list cnpg.io/Backups: %v", err) + } + if len(backups.Items) != 1 || backups.Items[0].Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { + t.Fatalf("cnpg.io/Backups = %+v, want exactly one, of bj, whose write came first", backups.Items) + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want bj's s3://bucket/ under its running Backup", got) + } + if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { + t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) + } +} + +// The claim names the job whose write last reached the ObjectStore, whether +// it changed the configuration, created the object or left it as it was; the +// change record is re-sent as it is by a write that changes nothing. +func TestReconcileCNPG_ClaimNamesTheJobThatWroteTheDestination(t *testing.T) { + t.Run("changed", func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.pass() + if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) + } + }) + t.Run("created", func(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + f.pass() + store := f.liveObjectStore() + if got := store.Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) + } + if _, ok := store.Annotations[cnpgDestinationChangedAtAnnotation]; ok { + t.Errorf("creating the ObjectStore stamped %s, which would hold the Backup for nothing", cnpgDestinationChangedAtAnnotation) + } + }) + for name, recorded := range map[string]map[string]string{ + "unchanged": {cnpgDestinationWrittenByAnnotation: "bj-old"}, + "unchanged with a stamp": {cnpgDestinationWrittenByAnnotation: "bj-old", cnpgDestinationChangedAtAnnotation: "2026-01-02T03:04:05Z"}, + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + store := f.liveObjectStore() + store.Annotations = recorded + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + var sent []map[string]string + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && !slices.Contains(po.DryRun, metav1.DryRunAll) { + sent = append(sent, obj.GetAnnotations()) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + f.pass() + if len(sent) != 1 { + t.Fatalf("got %d ObjectStore writes, want 1", len(sent)) + } + if got := sent[0][cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("the write sent %s = %q, want bj, the job that wrote", cnpgDestinationWrittenByAnnotation, got) + } + if got, want := sent[0][cnpgDestinationChangedAtAnnotation], recorded[cnpgDestinationChangedAtAnnotation]; got != want { + t.Errorf("the write sent %s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, want) + } + }) + } +} + +// A claim stops holding the destination once its job has finished or is +// gone, so the next job writes its own. +func TestReconcileCNPG_ClaimOfAFinishedJobDoesNotHold(t *testing.T) { + for name, phase := range map[string]backupsv1alpha1.BackupJobPhase{ + "succeeded": backupsv1alpha1.BackupJobPhaseSucceeded, + "failed": backupsv1alpha1.BackupJobPhaseFailed, + "gone": "", + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + if name != "gone" { + f.seedOtherBackupJob("bj-other", phase) + } + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, _ := f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want bj's s3://bucket/", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, held by a %s job", ready, name) + } + }) + } +} + +// A job that attaches the plugin writes a configuration the ObjectStore may +// already carry, for instance after a restore re-rendered the Cluster +// without the plugin. Its write still claims the destination, so a job with +// another one waits through the rollout instead of switching it. +func TestReconcileCNPG_UnchangedWriteClaimsTheDestinationForTheRollout(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) + store := &cnpgtypes.ObjectStore{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster, + Annotations: map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"}}, + Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: "s3://bucket/"}}, + } + if err := f.c.Create(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + pass2 := f.addJobWithDestination("bj2", "s3://other/") + + f.pass() // latch + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want bj waiting for the rollout of the plugin it attached", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + j2 := pass2() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, switched during bj's rollout", got) + } + if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { + t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) + } +} + +// The claim covers only the window before its job has a Backup the queue +// can see. Once it has one, the Backup in flight holds the destination, for +// as long as CloudNativePG runs it, and a Backup that finished does not, even +// if its job is stuck before it finishes, so that job cannot hold every later +// backup of the application for as long as it lives. +func TestReconcileCNPG_ClaimOfAJobWithItsOwnBackupDoesNotHold(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, cnpgBackupPhaseComplete) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, _ := f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, held by a job whose Backup already finished", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, held by a job whose Backup already finished", ready) + } +} + +// A claim whose job cannot be read cannot be taken as released: writing over +// it would let two jobs flip the destination again. The error goes back to +// the controller for a retry. +func TestReconcileCNPG_ClaimHolderLookupErrorIsReturned(t *testing.T) { + unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") + for name, setup := range map[string]func(f *cnpgRolloutFixture){ + "holder job read fails": func(f *cnpgRolloutFixture) { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.BackupJob); ok && key.Name == "bj-other" { + return unavailable + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }, + "holder Backups list fails": func(f *cnpgRolloutFixture) { + holders := labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj-other", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS} + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + if _, ok := list.(*cnpgtypes.BackupList); ok && lo.LabelSelector != nil && lo.LabelSelector.Matches(holders) && !lo.LabelSelector.Matches(labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS}) { + return unavailable + } + return cl.List(ctx, list, opts...) + }, + }) + }, + } { + t.Run(name, func(t *testing.T) { + assertClaimHolderLookupErrorReturned(t, setup) + }) + } +} + +func assertClaimHolderLookupErrorReturned(t *testing.T, setup func(f *cnpgRolloutFixture)) { + t.Helper() + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + setup(f) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the claim holder lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the claim holder could not be read: %q", got) + } +} + +// A running job that holds the claim keeps the destination while it has no +// Backup yet, for instance while the plugin it attached rolls the instances. +func TestReconcileCNPG_ClaimOfARunningJobHolds(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination = %q, rewritten under bj-other's claim", got) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created under another job's claim: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { + t.Errorf("Ready = %+v, want WaitingForOtherBackup naming bj-other", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a held job is not requeued: %+v", res) + } +} + +// The plugin sidecar reads the ObjectStore through a cache that nothing this +// controller writes invalidates, so a Backup created right after the +// destination changed can be taken against the previous one. The job holds +// the Backup until that cache has expired, and creates it afterwards. +func TestReconcileCNPG_BackupWaitsForThePluginCacheAfterADestinationChange(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Fatalf("destination = %q, want the job's s3://bucket/", got) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created in the pass that changed the destination: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForDestinationToSettle" { + t.Errorf("Ready = %+v, want WaitingForDestinationToSettle", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil || time.Since(stamp) > time.Minute { + t.Fatalf("%s = %v (%v), want the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) + } + + if _, backups = f.pass(); len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created while the plugin cache could still hold the old destination: %+v", backups) + } + + f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) + if j, backups = f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups once the cache expired, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } +} + +// A write that leaves the configuration as it was cannot leave a stale +// cache behind: the Backup is created at once, and the record of the last +// change is re-sent as it is, since an apply that left it out would delete it. +func TestReconcileCNPG_UnchangedDestinationDoesNotWait(t *testing.T) { + const stamp = "2026-01-02T03:04:05Z" + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + f.setDestinationStamp(stamp) + + if j, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups for an unchanged destination, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if got := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; got != stamp { + t.Errorf("%s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, stamp) + } +} + +// The apiserver answers an apply with the merged object: fields it defaults, +// or that another field manager owns, are there although the job never sets +// them. Judged against what the job sends, every pass would read as a change +// and the Backup would never be created; judged against the dry run of the +// same apply, an unchanged configuration stays unchanged. +func TestReconcileCNPG_FieldsTheJobDoesNotSetAreNotAChange(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + store := f.liveObjectStore() + store.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if os, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { + os.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} + return nil + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + if j, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups for a configuration only the server filled in, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { + t.Errorf("%s was set by a write that changed nothing", cnpgDestinationChangedAtAnnotation) + } +} + +// Without the live ObjectStore, or the dry run of the apply, the job cannot +// tell whether its write changes the destination, and taking the write as +// unchanged would create the Backup at once against a cache that may still +// hold the previous one. Either failure is retried like a failed write. +func TestReconcileCNPG_UnreadableObjectStoreIsRetried(t *testing.T) { + unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") + for _, tc := range []struct { + name string + setup func(f *cnpgRolloutFixture) + }{ + {name: "live read fails", setup: func(f *cnpgRolloutFixture) { + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return unavailable + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "claim read fails", setup: func(f *cnpgRolloutFixture) { + reads := 0 + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + if reads++; reads == 1 { + return unavailable + } + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "live read at the write fails", setup: func(f *cnpgRolloutFixture) { + reads := 0 + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + if reads++; reads > 1 { + return unavailable + } + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "dry run fails", setup: func(f *cnpgRolloutFixture) { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { + return unavailable + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + tc.setup(f) + assertObjectStoreReadRetried(t, f) + }) + } +} + +// assertObjectStoreReadRetried runs one pass and checks it is retried, either +// by returning the error to the controller or by waiting with +// ClusterPluginApplyRetrying, without a Backup, a destination write or a +// failed job. +func assertObjectStoreReadRetried(t *testing.T, f *cnpgRolloutFixture) { + t.Helper() + ctx := context.Background() + key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, key, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, reconcileErr := f.r.reconcileCNPG(ctx, j, f.resolved) + if err := f.c.Get(ctx, key, j); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + backups := &cnpgtypes.BackupList{} + if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { + t.Fatalf("list cnpg.io/Backups: %v", err) + } + if len(backups.Items) != 0 { + t.Fatalf("a cnpg.io/Backup was created although the ObjectStore could not be read: %+v", backups.Items) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination = %q, written without knowing whether it changed", got) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed on a transient read: %s", j.Status.Message) + } + if reconcileErr != nil { + return + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "ClusterPluginApplyRetrying" { + t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a retried read is not requeued: %+v", res) + } +} + +// A change record that cannot be read, or that lies further ahead than +// controller clocks drift, says nothing about when the cache expires, so the +// write replaces it with its own time and the job waits once. It claims the +// destination for that wait, or another job would write its own meanwhile. +func TestReconcileCNPG_UnusableDestinationRecordIsReplaced(t *testing.T) { + for name, value := range map[string]string{ + "does not parse": "yesterday", + "far ahead": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + f.setDestinationStamp(value) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created with an unusable change record: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed: %s", j.Status.Message) + } + stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil || time.Since(stamp) > time.Minute || time.Until(stamp) > time.Minute { + t.Fatalf("%s = %v (%v), want it replaced with the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) + } + if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj, which waits on the replaced record", cnpgDestinationWrittenByAnnotation, got) + } + }) + } +} + +// A later write to a Cluster that already carries the plugin re-sends the +// attach record as it is: moving it would make the instances that rolled +// with the plugin look older than the attach. +func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) { + const stamp = "2026-01-02T03:04:05Z" + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(time.Now().Add(-time.Hour))) + + var sent []map[string]string + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok { + sent = append(sent, obj.GetAnnotations()) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if _, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the one this pass creates", len(backups)) + } + if len(sent) != 1 { + t.Fatalf("got %d Cluster writes, want 1", len(sent)) + } + if got := sent[0][cnpgPluginAttachedAtAnnotation]; got != stamp { + t.Errorf("the write sent %s = %q, want the recorded %q", cnpgPluginAttachedAtAnnotation, got, stamp) + } +} + +// An attach record that does not parse cannot tell which instances rolled +// with the plugin, so the job fails naming it instead of guessing. One further +// ahead than controller clocks drift apart would hold every instance as older +// than the plugin and push the rollout deadline out by as far as it lies +// ahead, so it fails the same way. +func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { + for _, tc := range []struct{ name, record string }{ + {name: "does not parse", record: "yesterday"}, + {name: "in the future", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} + f.observeCNPG(live) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created with an unreadable attach record: %+v", backups) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgPluginAttachedAtAnnotation) { + t.Errorf("phase %q, message %q; want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgPluginAttachedAtAnnotation) + } + }) + } +} + +// A record a little ahead of this controller's clock is what a leadership +// change between controllers whose clocks differ leaves behind, so the job +// waits for the clocks to agree rather than failing an attach that worked. +func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups) + } + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } +} + +// A job that attached nothing itself still finds the plugin another backup +// job attached a moment ago, while CNPG rolls the instances for it; a Backup +// created then fails for good. It waits for that rollout as long as the +// attach is recent enough for it to be running, and an attach record it +// cannot read is not its own to fail on. +func TestReconcileCNPG_JobWithoutLatchWaitsForAnotherJobsRollout(t *testing.T) { + cases := []struct { + name string + record string + podCreated time.Time + wantBackup bool + }{ + {name: "recent attach, instance not rolled yet", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-10 * time.Minute), wantBackup: false}, + {name: "recent attach, instance rolled", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Minute), wantBackup: true}, + {name: "attach older than the rollout deadline", record: time.Now().Add(-2 * cnpgDefaultBackupDeadline).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-3 * cnpgDefaultBackupDeadline), wantBackup: true}, + {name: "unreadable record", record: "yesterday", podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + {name: "record further ahead than clocks drift", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + {name: "record ahead within clock drift", record: time.Now().Add(time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(tc.podCreated.Truncate(time.Second))) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed: %s", j.Status.Message) + } + if got := len(backups) == 1; got != tc.wantBackup { + t.Fatalf("cnpg.io/Backup created = %t, want %t; Ready = %+v", got, tc.wantBackup, apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if !tc.wantBackup { + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } + } + }) + } +} + +// A job whose own Backup already exists is past every wait before it: it +// tracks that Backup even while another job's recent attach is still rolling +// the instances, since waiting would only hold the result of a Backup that +// runs regardless. +func TestReconcileCNPG_JobWithItsOwnBackupDoesNotWaitForAnotherJobsRollout(t *testing.T) { + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339)} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(time.Now().Add(-10*time.Minute).Truncate(time.Second))) + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready = %+v, want WaitingForCNPGBackup tracking the job's own Backup", ready) + } +} + +// A plugin attached by someone else after the latch, another backup job or +// the chart re-rendering a restored Cluster outside the platform flow, +// started the rollout before this job's write, which only finds it there: +// instances that rolled since the latch carry the plugin. +func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) { + oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))) + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(5*time.Minute))) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups against instances that rolled with the plugin; Ready = %+v", + len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } +} + +// Waiting for another backup resets the retry budget of the destination +// write, so the queue does not spend it. +func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond) + } + + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + j, _ = f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the first failure after the queue failed the job: %s", j.Status.Message) + } +} + +// seedRestoreOfApp starts a RestoreJob that writes into the fixture's app. +func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob { + f.t.Helper() + appGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-backup"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &appGroup, Kind: postgresAppKind, Name: rolloutApp}, + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, + }, + } + restore := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-restore"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + } + for _, obj := range []client.Object{backup, restore} { + if err := f.c.Create(context.Background(), obj); err != nil { + f.t.Fatalf("seed %T: %v", obj, err) + } + } + return restore +} + +// The restore check before the destination write reads the apiserver: the +// cache can still miss a RestoreJob created a moment ago, and a write landing +// in that gap can put the restored Cluster back on the source's WAL prefix. +func TestReconcileCNPG_RestoreCheckReadsPastTheCache(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedRestoreOfApp() + f.r.APIReader = f.c + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return nil + } + return cl.List(ctx, list, opts...) + }, + }) + + j, backups := f.pass() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written during a restore the cache had not caught: %q", got) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created during a restore the cache had not caught: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { + t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) + } +} + +// A job past its start writes the WAL archive destination only while no +// restore rewrites its application. The restore gate in Reconcile looks only +// at a job that has not started, and a job released from the queue, or one +// still before its first write, has started long ago: a write landing during +// the restore can put the restored Cluster back on the source's WAL prefix. +func TestReconcileCNPG_HoldsTheDestinationWriteWhileTheAppIsRestored(t *testing.T) { + for _, queued := range []bool{true, false} { + t.Run(fmt.Sprintf("released from the queue=%t", queued), func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + if queued { + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForOtherBackup" { + t.Fatalf("Ready = %+v, want the job queued", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + } + restore := f.seedRestoreOfApp() + if queued { + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + holder := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other"}, holder); err != nil { + t.Fatalf("get holder BackupJob: %v", err) + } + holder.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), holder); err != nil { + t.Fatalf("finish holder BackupJob: %v", err) + } + } + + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written during a restore of the app: %q", got) + } + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Errorf("a cnpg.io/Backup was created during a restore of the app: %s", b.Name) + } + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { + t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) + } + if res.RequeueAfter == 0 { + t.Errorf("a held job is not requeued: %+v", res) + } + + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("once the restore finished, destination = %q, want the job's own s3://bucket/", got) + } + }) + } +} + +// A job whose Backup already runs writes nothing, so a restore does not hold +// it: holding would strand a backup that is already moving data. +func TestReconcileCNPG_RestoreDoesNotHoldARunningBackup(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.seedRestoreOfApp() + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready = %+v, want the running backup still followed", ready) + } +} + +// An error looking for a restore is not taken as "no restore". +func TestReconcileCNPG_RestoreLookupErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.List(ctx, list, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the restore lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although restores could not be listed: %q", got) + } +} + +// Waiting for a restore resets the retry budget of the destination write, +// as waiting for another backup does, so the restore does not spend it. +func TestReconcileCNPG_RestoreResetsDestinationRetryBudget(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + restore := f.seedRestoreOfApp() + + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Fatalf("%s = %+v kept while waiting for a restore", backupCondDestinationApplied, cond) + } + + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the first failure after the restore failed the job: %s", j.Status.Message) + } +} + +// With two jobs holding the queue the wait names the same one on every pass, +// whatever order the cache lists their Backups in, so a waiting job does not +// rewrite its status each poll. +func TestReconcileCNPG_QueueNamesOneHolderAcrossPasses(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedOtherBackupJob("holder-z", backupsv1alpha1.BackupJobPhaseRunning) + f.seedOtherBackupJob("holder-a", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("a-cnpg", "holder-z", rolloutCluster, "running") + f.seedCNPGBackup("z-cnpg", "holder-a", rolloutCluster, "running") + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "holder-a ") { + t.Fatalf("Ready = %+v, want WaitingForOtherBackup naming holder-a", ready) + } + statusWrites = 0 + f.pass() + if statusWrites != 0 { + t.Errorf("a second pass with the same two holders wrote the BackupJob status %d time(s)", statusWrites) + } +} + +func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + holder, err := f.r.otherCNPGBackupInFlight(context.Background(), j, rolloutCluster) + if err != nil { + t.Fatalf("otherCNPGBackupInFlight: %v", err) + } + if holder != "" { + t.Fatalf("the job waits on its own Backup: holder %q", holder) + } +} + +// Once the job's Backup exists the Cluster is only read; a failed read is +// returned for a retry and neither fails the running backup nor starts the +// destination retry budget. +func TestReconcileCNPG_ClusterReadErrorDoesNotFailRunningBackup(t *testing.T) { + for name, readErr := range map[string]error{ + "internal error": apierrors.NewInternalError(errors.New("blip")), + // A write answered Forbidden fails the job; a read must not. + "forbidden": apierrors.NewForbidden(schema.GroupResource{Group: cnpgtypes.GroupName, Resource: "clusters"}, rolloutCluster, errors.New("denied")), + } { + t.Run(name, func(t *testing.T) { testClusterReadErrorKeepsBackup(t, readErr) }) + } +} + +func testClusterReadErrorKeepsBackup(t *testing.T, readErr error) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + dyn := f.r.Interface.(*dynamicfake.FakeDynamicClient) + dyn.PrependReactor("get", "clusters", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, readErr + }) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("a failed Cluster read was not returned for a retry") + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a failed Cluster read failed the running backup: %s", persisted.Status.Message) + } +} + // cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose // Cluster and instance pods a test swaps between passes, standing in for // CNPG rolling the instances. @@ -2409,6 +4262,15 @@ func newRolloutInstancePod(created time.Time) *corev1.Pod { // pass runs one reconcile and returns the persisted BackupJob and the // cnpg.io/Backups that exist afterwards. func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup) { + f.t.Helper() + j, backups, _ := f.passResult() + return j, backups +} + +// passResult is pass that also returns the reconcile result: the controller +// watches only BackupJobs, so a pass that waits on anything else is woken +// only by its own requeue. +func (f *cnpgRolloutFixture) passResult() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup, ctrl.Result) { f.t.Helper() ctx := context.Background() key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} @@ -2416,7 +4278,8 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac if err := f.c.Get(ctx, key, j); err != nil { f.t.Fatalf("get BackupJob: %v", err) } - if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil { + res, err := f.r.reconcileCNPG(ctx, j, f.resolved) + if err != nil { f.t.Fatalf("reconcileCNPG: %v", err) } got := &backupsv1alpha1.BackupJob{} @@ -2427,7 +4290,7 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { f.t.Fatalf("list cnpg.io/Backups: %v", err) } - return got, backups.Items + return got, backups.Items, res } // observeCNPG makes the next live read see cluster, as CNPG has written it. @@ -2516,8 +4379,8 @@ func TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup(t *testing.T) { if !apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { t.Errorf("%s condition = %+v, want True", backupCondPluginRolledOut, apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)) } - if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond != nil { - t.Errorf("Ready condition = %+v left over from the wait while the backup runs", cond) + if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready condition = %+v, want the wait on the cnpg.io/Backup rather than one left over from the rollout", cond) } } @@ -2536,6 +4399,76 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) { if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond != nil { t.Errorf("%s condition = %+v on a Cluster the driver did not roll", backupCondPluginRolledOut, cond) } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); cond != nil { + t.Errorf("%s condition = %+v on a job nothing held", backupCondDestinationWriteHeld, cond) + } +} + +// An in-place restore of the platform flow recreates the Cluster without the +// barman-cloud plugin, so a job held through it attaches the plugin again +// once released. That attach starts a rollout of its own, which the +// instances restored after the job's first rollout began have not done. +func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) { + stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(stampedAt), + }} + f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase), + newRolloutInstancePod(time.Now().Add(-2*time.Minute))) + restore := f.seedRestoreOfApp() + if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress { + t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready")) + } + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + + attachAt := time.Now().Truncate(time.Second) + got, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups) + } + rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut) + if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { + t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) + } +} + +// A release older than the rollout, from a wait before the plugin was +// attached, does not shorten the rollout that began after it. +func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + j := newRolloutBackupJob(expired.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{ + { + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionFalse, + Reason: "Released", + LastTransitionTime: metav1.NewTime(expired), + }, + { + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)), + }, + } + f := newCNPGRolloutFixture(t, j, + newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(expired.Add(-time.Hour))) + + got, backups := f.pass() + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that began a minute ago failed on an older release: %s", got.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups) + } } // TestReconcileCNPG_PluginRolloutWaitIsBounded: a rollout that never ends @@ -2565,6 +4498,142 @@ func TestReconcileCNPG_PluginRolloutWaitIsBounded(t *testing.T) { } } +// A job whose plugin rollout started, and which then waited for another +// backup of the app or for a restore of it, gets the rollout deadline back +// from the moment it was released: that wait is not a rollout of its own. +// The instance pods still count as rolled from the rollout's own start. +func TestReconcileCNPG_HeldJobRolloutDeadlineCountsFromRelease(t *testing.T) { + type hold struct { + name string + seed func(f *cnpgRolloutFixture) (release func()) + waiting string + } + holds := []hold{ + { + name: "another backup", + waiting: "WaitingForOtherBackup", + seed: func(f *cnpgRolloutFixture) func() { + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + return func() { + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + f.t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + f.t.Fatalf("complete holder Backup: %v", err) + } + } + }, + }, + { + name: "a restore", + waiting: ConditionReasonRestoreInProgress, + seed: func(f *cnpgRolloutFixture) func() { + restore := f.seedRestoreOfApp() + return func() { + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + f.t.Fatalf("finish the restore: %v", err) + } + } + }, + }, + } + stampedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + newHeldRollout := func(t *testing.T, h hold, cluster *cnpgtypes.Cluster, pod *corev1.Pod) *cnpgRolloutFixture { + t.Helper() + j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(stampedAt), + }} + f := newCNPGRolloutFixture(t, j, cluster, pod) + release := h.seed(f) + got, backups := f.pass() + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Fatalf("a held job created cnpg.io/Backup %s", b.Name) + } + } + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a held job did not wait: %s", got.Status.Message) + } + if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != h.waiting { + t.Fatalf("Ready = %+v, want %s", ready, h.waiting) + } + release() + return f + } + + for _, h := range holds { + t.Run(h.name+": the rollout is still waited for after the release", func(t *testing.T) { + f := newHeldRollout(t, h, + newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(stampedAt.Add(-time.Hour))) + + releasedAt := time.Now().Truncate(time.Second) + j, backups := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("time held by %s was charged to the rollout: %s", h.name, j.Status.Message) + } + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Fatalf("cnpg.io/Backup %s was created against an unfinished rollout", b.Name) + } + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want WaitingForPluginRollout", ready) + } + released := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) + if released == nil || released.Status != metav1.ConditionFalse || released.LastTransitionTime.Time.Before(releasedAt) { + t.Fatalf("%s = %+v, want False since the release at %s", backupCondDestinationWriteHeld, released, releasedAt) + } + if rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); rollout == nil || !rollout.LastTransitionTime.Time.Equal(stampedAt) { + t.Errorf("%s = %+v, want its stamp kept at %s", backupCondPluginRolledOut, rollout, stampedAt) + } + + released.LastTransitionTime = metav1.NewTime(stampedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the release: %v", err) + } + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed once the rollout outlived the deadline after the release", j.Status.Phase, j.Status.Message) + } + }) + + t.Run(h.name+": an instance that rolled while the job was held counts", func(t *testing.T) { + f := newHeldRollout(t, h, + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(stampedAt.Add(time.Minute))) + + var j *backupsv1alpha1.BackupJob + own := 0 + for range 3 { + var backups []cnpgtypes.Backup + j, backups = f.pass() + own = 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + own++ + } + } + if own != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if own != 1 { + t.Fatalf("got %d cnpg.io/Backups of the job against an instance that rolled after the rollout began; Ready = %+v, phase %q (%s)", + own, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"), j.Status.Phase, j.Status.Message) + } + }) + } +} + // TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline covers the // restore-path blocker: a RestoreJob whose referenced CNPG Strategy CR never // appears must NOT requeue forever. requeueRestoreStrategyNotReady gates @@ -4188,7 +6257,7 @@ func TestCreateCNPGBackupArtifact_RecordsMajorVersion(t *testing.T) { rendered := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}, } - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } @@ -4369,3 +6438,339 @@ func TestReconcileCNPGRestore_RefusesCrossMajorRestore(t *testing.T) { }) } } + +func (f *cnpgRolloutFixture) setStrategyDestination(destination, endpoint, secret string) { + f.t.Helper() + strategy := &strategyv1alpha1.CNPG{} + if err := f.c.Get(context.Background(), client.ObjectKey{Name: "cnpg-strategy"}, strategy); err != nil { + f.t.Fatalf("get strategy: %v", err) + } + strategy.Spec.Template.BarmanObjectStore.DestinationPath = destination + strategy.Spec.Template.BarmanObjectStore.EndpointURL = endpoint + strategy.Spec.Template.BarmanObjectStore.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{SecretRef: corev1.LocalObjectReference{Name: secret}} + if err := f.c.Update(context.Background(), strategy); err != nil { + f.t.Fatalf("update strategy: %v", err) + } +} + +// completeOwnBackup applies edit, when given, to the job's cnpg.io/Backup and +// marks it completed. +func (f *cnpgRolloutFixture) completeOwnBackup(edit func(b *cnpgtypes.Backup)) { + f.t.Helper() + backups := &cnpgtypes.BackupList{} + if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { + f.t.Fatalf("list cnpg.io/Backups: %v", err) + } + for i := range backups.Items { + b := &backups.Items[i] + if b.Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { + continue + } + if edit != nil { + edit(b) + } + b.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), b); err != nil { + f.t.Fatalf("complete cnpg.io/Backup: %v", err) + } + return + } + f.t.Fatalf("the job has no cnpg.io/Backup") +} + +func (f *cnpgRolloutFixture) artifact() *backupsv1alpha1.Backup { + f.t.Helper() + artifact := &backupsv1alpha1.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, artifact); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + f.t.Fatalf("get Backup artifact: %v", err) + } + return artifact +} + +// The destination is written by the pass that creates the cnpg.io/Backup and +// not again while it runs, so the artifact has to name what that pass wrote: +// a strategy edited while the backup runs does not reach the backup, and must +// not reach its artifact either. The BackupJob, which a tenant can edit, +// carries no part of it. +func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.setStrategyDestination("s3://bucket/", "https://s3.one", "creds-one") + + f.pass() + f.settleDestination() + j, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups once the destination settled, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Fatalf("destination = %q, want s3://bucket/", got) + } + want := `{"destinationPath":"s3://bucket/","endpointURL":"https://s3.one","s3SecretRef":"creds-one","serverName":"` + rolloutCluster + `"}` + if got := backups[0].Annotations[cnpgWrittenDestinationAnnotation]; got != want { + t.Fatalf("%s on the created cnpg.io/Backup = %q, want %q", cnpgWrittenDestinationAnnotation, got, want) + } + + f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") + j.Annotations = map[string]string{cnpgWrittenDestinationAnnotation: `{"destinationPath":"s3://tenant/","endpointURL":"https://tenant","s3SecretRef":"creds-one"}`} + if err := f.c.Update(context.Background(), j); err != nil { + t.Fatalf("annotate BackupJob: %v", err) + } + f.completeOwnBackup(nil) + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if artifact == nil { + t.Fatalf("no Backup artifact") + } + md := artifact.Spec.DriverMetadata + if md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "https://s3.one" || md[cnpgS3SecretRefKey] != "creds-one" { + t.Fatalf("artifact driver metadata = %v, want the destination written with the Backup (s3://bucket/, https://s3.one, creds-one)", md) + } +} + +// The serverName is the archive prefix under the destination, so the record +// carries it too: a restore that re-renders the Cluster between the backup +// and its artifact gives the live Cluster another one. A record written +// before it carried a serverName falls back to the live Cluster. +func TestReconcileCNPG_ArtifactNamesTheServerNameItsBackupWasCreatedWith(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + _, backups := f.pass() + want := `{"destinationPath":"s3://bucket/","serverName":"` + rolloutCluster + `"}` + if len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != want { + t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording %s", backups, want) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "restored-server"))) + f.completeOwnBackup(nil) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if got := artifact.Spec.DriverMetadata[cnpgServerNameKey]; got != rolloutCluster { + t.Errorf("artifact serverName = %q, want %q recorded with the Backup", got, rolloutCluster) + } + if !strings.HasPrefix(artifact.Status.Artifact.URI, "cnpg://"+rolloutCluster+"/") { + t.Errorf("artifact URI = %q, want it under %s", artifact.Status.Artifact.URI, rolloutCluster) + } +} + +func TestReconcileCNPG_ArtifactOfARecordWithoutServerName(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.pass() + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "live-server"))) + f.completeOwnBackup(func(b *cnpgtypes.Backup) { + b.Annotations[cnpgWrittenDestinationAnnotation] = `{"destinationPath":"s3://bucket/"}` + }) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + if got := f.artifact().Spec.DriverMetadata[cnpgServerNameKey]; got != "live-server" { + t.Errorf("artifact serverName = %q, want the live Cluster's live-server", got) + } +} + +// A strategy without credentials records none, and the artifact names none. +func TestReconcileCNPG_ArtifactOfARecordWithoutCredentials(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + if _, backups := f.pass(); len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != `{"destinationPath":"s3://bucket/","serverName":"`+rolloutCluster+`"}` { + t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording only s3://bucket/", backups) + } + f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") + f.completeOwnBackup(nil) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + md := f.artifact().Spec.DriverMetadata + if _, ok := md[cnpgS3SecretRefKey]; ok || md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "" { + t.Fatalf("artifact driver metadata = %v, want s3://bucket/ with no endpoint and no credentials", md) + } +} + +// A record that names no usable destination cannot say where the backup is, +// so no artifact is created rather than one pointing at the strategy as it +// reads now. +func TestReconcileCNPG_ArtifactRefusedForAnUnusableDestinationRecord(t *testing.T) { + for name, value := range map[string]string{ + "empty": "", + "unreadable": "{not json", + "not an object": `"s3://bucket/"`, + "null": "null", + "no fields": "{}", + "empty destination": `{"destinationPath":"","endpointURL":"https://s3.one"}`, + // Decoding sets the destination path and skips the mistyped + // endpoint, which would leave the artifact on the default one. + "mistyped endpoint": `{"destinationPath":"s3://bucket/","endpointURL":7}`, + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + if _, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + f.completeOwnBackup(func(b *cnpgtypes.Backup) { + if b.Annotations == nil { + b.Annotations = map[string]string{} + } + b.Annotations[cnpgWrittenDestinationAnnotation] = value + }) + j, _ := f.pass() + if f.artifact() != nil { + t.Fatalf("an artifact was created from the record %q", value) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgWrittenDestinationAnnotation) { + t.Fatalf("phase = %q (%s), want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgWrittenDestinationAnnotation) + } + }) + } +} + +// A cnpg.io/Backup created by a controller that recorded nothing keeps +// getting its artifact from the strategy. +func TestReconcileCNPG_ArtifactOfABackupWithoutARecord(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.seedCNPGBackup("bj-legacy", "bj", rolloutCluster, cnpgBackupPhaseComplete) + + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if artifact == nil || artifact.Spec.DriverMetadata[cnpgDestinationPathKey] != "s3://bucket/" { + t.Fatalf("artifact = %+v, want one built from the strategy", artifact) + } +} + +// cacheLaggingBehind makes the reconciler's cached client miss the named +// cnpg.io/Backups, as an informer does right after they are created, while +// its uncached reader sees them. +func (f *cnpgRolloutFixture) cacheLaggingBehind(hidden ...string) { + f.r.APIReader = f.c + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if backups, ok := list.(*cnpgtypes.BackupList); ok { + backups.Items = slices.DeleteFunc(backups.Items, func(b cnpgtypes.Backup) bool { + return slices.Contains(hidden, b.Name) + }) + } + return nil + }, + }) +} + +// Reconciles run one at a time, so a Backup created by one pass exists by +// the next; the queue has to see it even when the cache does not yet, or +// the next job rewrites the destination under it and the record on that +// Backup no longer names where its WAL goes. +func TestReconcileCNPG_QueueSeesABackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + f.cacheLaggingBehind("bj-other-cnpg") + + j, backups := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" { + t.Fatalf("Ready = %+v, want WaitingForOtherBackup", ready) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the other job's Backup", got) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want only the other job's", len(backups)) + } +} + +// The job's own Backup, missing from the cache right after it was created, +// is neither created again nor written under. +func TestReconcileCNPG_OwnBackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.cacheLaggingBehind("bj-cnpg") + + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the job's one", len(backups)) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the job's own Backup", got) + } +} + +// failingBackupLists makes the uncached reader fail the cnpg.io/Backup lists +// for which fail, given whether the list selects only the job's own Backup +// (it would not find another job's) and how many such lists came before, +// returns true. +func (f *cnpgRolloutFixture) failingBackupLists(fail func(own bool, n int) bool) { + counts := map[bool]int{} + anotherJobs := labels.Set{ + backupsv1alpha1.OwningJobNameLabel: "bj-other", + backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, + cnpgClusterLabel: rolloutCluster, + } + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cnpgtypes.BackupList); ok { + o := &client.ListOptions{} + o.ApplyOptions(opts) + own := o.LabelSelector != nil && !o.LabelSelector.Matches(anotherJobs) + counts[own]++ + if fail(own, counts[own]) { + return apierrors.NewServiceUnavailable("apiserver unavailable") + } + } + return cl.List(ctx, list, opts...) + }, + }) +} + +// A failed apiserver read of the Backups is retried, not taken as the job +// failing: nothing has been written yet that the retry would not redo. +func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) { + for name, own := range map[string]bool{"own Backup": true, "queue": false} { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.failingBackupLists(func(o bool, _ int) bool { return o == own }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the Backup list error") + } + got := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, got); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed on a Backup list error: %s", got.Status.Message) + } + if dest := f.liveDestination(); dest != "s3://other/" { + t.Fatalf("destination = %q, written although the Backups could not be read", dest) + } + }) + } +} + +// The pass that writes the destination looks its own Backup up once, at the +// start: a second lookup before the create could only fail, after the write, +// and fail the job with it. +func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 }) + + j, backups := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) + } +} diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index 4e0c3b75a..aaba89b3c 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -1,11 +1,14 @@ package backupcontroller import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "maps" "net/url" + "strconv" "strings" "time" @@ -15,6 +18,8 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" ) // CredentialsProjectionRequeue is the backoff between reconcile attempts @@ -73,9 +78,8 @@ func IsTransient(err error) bool { // BackupCredentialsConfig describes the platform-managed source Secret // produced by the cozy-backups Bucket, the per-tenant projection target // Secret name, and the bucket coordinates (endpoint used to format the -// FDB blob_credentials account-host key, region carried verbatim for -// drivers like clickhouse-backup that wire it through env, forcePathStyle -// projected as a flat key so the chart-emitted ClickHouse sidecar can read +// FDB blob_credentials account-host key, region carried verbatim, +// forcePathStyle projected so the chart-emitted ClickHouse sidecar can read // a platform value it cannot see directly). Values are wired from // environment variables in main() and the deployment chart populates them // from .Values.backupStorage. @@ -85,11 +89,11 @@ type BackupCredentialsConfig struct { TargetSecretName string Endpoint string Region string - // ForcePathStyle is the string "true"/"false" from - // backupStorage.forcePathStyle, projected into the credentials Secret - // under the forcePathStyle key. The app charts cannot read - // backupStorage.* directly, so the ClickHouse sidecar consumes it via - // an optional secretKeyRef as S3_FORCE_PATH_STYLE. + // ForcePathStyle is backupStorage.forcePathStyle as the chart + // normalises it ("true", "false", or "" when unset), projected into the credentials Secret + // under the forcePathStyle key and into clickhouse-backup.yml. The app + // charts cannot read backupStorage.* directly, so the ClickHouse + // sidecar takes it from there. ForcePathStyle string } @@ -107,10 +111,11 @@ func (c BackupCredentialsConfig) IsEnabled() bool { // the data so every default Strategy CR can consume the same Secret: // // - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — CNPG, MariaDB, Etcd. -// - accessKey / secretKey + bucketName / endpoint / region — ClickHouse -// sidecar. +// - accessKey / secretKey + bucketName / endpoint / region — Redis +// strategy, and ClickHouse sidecars rendered before clickhouse-backup.yml. // - cloud — Velero credentials file ([default] section). // - blob_credentials.json — FoundationDB backup_agent. +// - clickhouse-backup.yml — ClickHouse sidecar, mounted as its config file. // // The source Secret is read in two formats: // @@ -156,8 +161,8 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if creds.endpoint == "" { // Scheme-strip the env-var fallback to match the source-Secret path // (parseSourceSecret strips there). The projected `endpoint` key is - // consumed verbatim as S3_ENDPOINT by the ClickHouse sidecar, which - // expects a bare host:port — see https://cozystack.io/docs/v1/operations/services/backup-classes/. + // a bare host:port and each consumer adds its own scheme — see + // https://cozystack.io/docs/v1/operations/services/backup-classes/. creds.endpoint = stripScheme(cfg.Endpoint) } if creds.endpoint == "" { @@ -177,6 +182,20 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr } cloud := buildVeleroCredentialsFile(creds.accessKey, creds.secretKey) + clickHouseConfig, pathStyleErr, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle) + if pathStyleErr != nil { + // The chart refuses such a value at render time, so only an env set + // around it gets here. It concerns the ClickHouse file alone, and + // failing the projection would take every other driver's + // credentials down with it. + log.FromContext(ctx).Error(pathStyleErr, "left force_path_style out of "+clickHouseBackupConfigKey, "namespace", targetNamespace) + } + if err != nil { + return &ProjectionError{ + Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("build %s: %v", clickHouseBackupConfigKey, err), + } + } blob, err := buildFDBBlobCredentials(creds.endpoint, creds.accessKey, creds.secretKey) if err != nil { return &ProjectionError{ @@ -231,28 +250,36 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if target.Data == nil { target.Data = map[string][]byte{} } + before := maps.Clone(target.Data) target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey) target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey) - // Mirror under the legacy bucket-controller key names so - // chart-emitted sidecars (clickhouse-backup) can consume the - // same Secret without a separate field map. + // Mirror under the legacy bucket-controller key names so the Redis + // strategy and older ClickHouse charts can consume the same Secret + // without a separate field map. target.Data["accessKey"] = []byte(creds.accessKey) target.Data["secretKey"] = []byte(creds.secretKey) target.Data["cloud"] = []byte(cloud) target.Data["blob_credentials.json"] = blob + target.Data[clickHouseBackupConfigKey] = clickHouseConfig // Always overwrite endpoint/bucket/region (delete if empty in // source) so a re-projection cannot leave stale values lingering - // after the bucket-controller rotates the source. Consumers - // (ClickHouse sidecar via secretKeyRef) must see exactly what the - // source carries, not a half-stale Secret. + // after the bucket-controller rotates the source. Consumers must + // see exactly what the source carries, not a half-stale Secret. setOrDelete(target.Data, "endpoint", creds.endpoint) setOrDelete(target.Data, "bucketName", creds.bucket) setOrDelete(target.Data, "region", creds.region) // forcePathStyle comes from the platform config (backupStorage. - // forcePathStyle), not the source Secret: the ClickHouse sidecar - // reads it as S3_FORCE_PATH_STYLE because the app chart cannot see + // forcePathStyle), not the source Secret: ClickHouse sidecars + // rendered before clickhouse-backup.yml read it as + // S3_FORCE_PATH_STYLE because the app chart cannot see // backupStorage.* directly. setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle) + if !maps.EqualFunc(before, target.Data, bytes.Equal) { + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + target.Annotations[credentialsUpdatedAtAnnotation] = time.Now().UTC().Format(time.RFC3339) + } return nil }); err != nil { // If the mutator refused to overwrite an unowned Secret, surface @@ -414,6 +441,48 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err return json.Marshal(doc) } +// credentialsUpdatedAtAnnotation records when the projected content last +// changed. A consumer that mounts the Secret as a volume sees a change only +// once kubelet rewrites the volume, so it uses this to tell how fresh its +// copy can be. +const credentialsUpdatedAtAnnotation = "backups.cozystack.io/credentials-updated-at" + +// clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup +// config. The ClickHouse chart mounts it as the sidecar's config file +// instead of passing the values as environment: clickhouse-backup reloads +// the file in each API handler that reaches S3, among them the +// create_remote, restore_remote and list calls the strategy makes, and +// kubelet rewrites a mounted Secret in place, so credentials projected +// after the pod started, or rotated since, reach a running sidecar. +const clickHouseBackupConfigKey = "clickhouse-backup.yml" + +// buildClickHouseBackupConfig renders that file. The projected endpoint is +// a bare host, and clickhouse-backup's S3 client rejects one without a +// scheme. A forcePathStyle that is not a boolean is left out of the file +// and reported in pathStyleErr; err is a failure to render the file at all. +func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) (config []byte, pathStyleErr, err error) { + s3 := map[string]any{ + "bucket": creds.bucket, + "endpoint": "https://" + creds.endpoint, + "access_key": creds.accessKey, + "secret_key": creds.secretKey, + } + // An empty region in the file would override clickhouse-backup's own + // us-east-1 default instead of leaving it in place. + if creds.region != "" { + s3["region"] = creds.region + } + if forcePathStyle != "" { + if pathStyle, perr := strconv.ParseBool(forcePathStyle); perr != nil { + pathStyleErr = fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle) + } else { + s3["force_path_style"] = pathStyle + } + } + config, err = yaml.Marshal(map[string]any{"s3": s3}) + return config, pathStyleErr, err +} + // stripScheme returns the endpoint stripped of any leading scheme prefix // (http:// or https://) and preserves the path portion if present. The // resulting form is what `host[:port][/path]` callers (MariaDB strategy, diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index 2b4f0bb1c..74fd936d1 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -5,7 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "strings" "testing" + "time" + + "github.com/go-logr/logr/funcr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,6 +17,8 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" ) func newFakeClient(objs ...client.Object) client.Client { @@ -664,3 +670,188 @@ func TestProject_HappyPath_HTTPEndpoint(t *testing.T) { t.Errorf("blob account host not stripped: %+v", blob.Accounts) } } + +// The ClickHouse sidecar reads its S3 settings from this key as a mounted +// file, which kubelet refreshes in place: clickhouse-backup reloads its +// config file in each API handler that reaches S3, while its environment +// is fixed at start. +func TestProject_WritesClickHouseBackupConfig(t *testing.T) { + cases := []struct { + name string + forcePathStyle string + wantPathStyle *bool + }{ + {name: "path style on", forcePathStyle: "true", wantPathStyle: ptrBool(true)}, + {name: "path style off", forcePathStyle: "false", wantPathStyle: ptrBool(false)}, + {name: "path style unset", forcePathStyle: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.ForcePathStyle = tc.forcePathStyle + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v\n%s", clickHouseBackupConfigKey, err, got.Data[clickHouseBackupConfigKey]) + } + for key, want := range map[string]string{ + "bucket": "cozy-backups", + "endpoint": "https://seaweedfs-s3.tenant-root.svc:8333", + "region": "us-east-1", + "access_key": "AK", + "secret_key": "SK", + } { + if doc.S3[key] != want { + t.Errorf("s3.%s = %v, want %q", key, doc.S3[key], want) + } + } + pathStyle, set := doc.S3["force_path_style"] + switch { + case tc.wantPathStyle == nil && set: + t.Errorf("s3.force_path_style = %v, want it left to the clickhouse-backup default", pathStyle) + case tc.wantPathStyle != nil && pathStyle != *tc.wantPathStyle: + t.Errorf("s3.force_path_style = %v, want %v", pathStyle, *tc.wantPathStyle) + } + }) + } +} + +// clickhouse-backup defaults the S3 region to us-east-1; an empty value in +// its config file would replace that default rather than fall back to it. +func TestProject_ClickHouseBackupConfigOmitsAnEmptyRegion(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.Region = "" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if region, set := doc.S3["region"]; set { + t.Errorf("s3.region = %q, want it left to the clickhouse-backup default", region) + } +} + +// The parse only serves the ClickHouse file, while the same projection +// carries every other driver's credentials. An unparseable platform value +// must cost ClickHouse its path-style setting and say so, not fail the +// projection for all of them. +func TestProject_MalformedForcePathStyleSkipsOnlyTheClickHouseSetting(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + var logged []string + ctx := log.IntoContext(context.Background(), funcr.New(func(prefix, args string) { + logged = append(logged, args) + }, funcr.Options{})) + cfg := defaultCfg() + cfg.ForcePathStyle = "yes" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("projection failed for every driver over a ClickHouse-only setting: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + if string(got.Data["AWS_ACCESS_KEY_ID"]) != "AK" { + t.Errorf("AWS_ACCESS_KEY_ID = %q, want the projected key", got.Data["AWS_ACCESS_KEY_ID"]) + } + if string(got.Data["forcePathStyle"]) != "yes" { + t.Errorf("forcePathStyle = %q, want the platform value carried verbatim", got.Data["forcePathStyle"]) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if doc.S3["access_key"] != "AK" { + t.Errorf("s3.access_key = %v, want the rest of the file written", doc.S3["access_key"]) + } + if pathStyle, set := doc.S3["force_path_style"]; set { + t.Errorf("s3.force_path_style = %v, want it left out", pathStyle) + } + found := false + for _, line := range logged { + if strings.Contains(line, `"error"`) && strings.Contains(line, `\"yes\"`) && strings.Contains(line, "force_path_style") { + found = true + } + } + if !found { + t.Errorf("no error naming the rejected value and the skipped setting was logged; got %q", logged) + } +} + +func ptrBool(b bool) *bool { return &b } + +// Consumers that read the Secret through a mounted volume need to know when +// its content last changed; an unchanged re-projection must keep the time. +func TestProject_StampsWhenContentChanges(t *testing.T) { + src := flatSourceSecret() + c := newFakeClient(src) + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"} + stamp := func() string { + got := &corev1.Secret{} + if err := c.Get(ctx, key, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + return got.Annotations[credentialsUpdatedAtAnnotation] + } + + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("first projection: %v", err) + } + first := stamp() + at, err := time.Parse(time.RFC3339, first) + if err != nil || time.Since(at) > time.Minute { + t.Fatalf("first projection stamp %q is not a recent RFC3339 time: %v", first, err) + } + + old := &corev1.Secret{} + if err := c.Get(ctx, key, old); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + old.Annotations[credentialsUpdatedAtAnnotation] = "2000-01-01T00:00:00Z" + if err := c.Update(ctx, old); err != nil { + t.Fatalf("age the stamp: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("unchanged re-projection: %v", err) + } + if got := stamp(); got != "2000-01-01T00:00:00Z" { + t.Fatalf("an unchanged re-projection moved the stamp to %q", got) + } + + rotated := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: src.Namespace, Name: src.Name}, rotated); err != nil { + t.Fatalf("get source Secret: %v", err) + } + rotated.Data["secretKey"] = []byte("SK2") + if err := c.Update(ctx, rotated); err != nil { + t.Fatalf("rotate source Secret: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("rotated re-projection: %v", err) + } + if got := stamp(); got == "2000-01-01T00:00:00Z" { + t.Fatalf("a rotation did not move the stamp") + } +} diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 9aea96f31..94075651e 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -593,12 +593,13 @@ func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *ba group = *strategy.StrategyRef.APIGroup } // When the Velero BSL is disabled (velero.bslEnabled=false, surfaced - // here as an empty VeleroNamespace) the chart gates the Velero - // Strategy CRs off the SAME flag, so they are never rendered. The - // BackupClass still routes to them unconditionally, so counting them - // as missing would force a Helm upgrade every MinForceInterval - // forever against a render that can never produce them. - if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" { + // here as an empty VeleroNamespace) the chart gates the Strategy CRs + // that back up through Velero (Velero, Kubernetes) off the SAME flag, + // so they are never rendered. The BackupClass still routes to them + // unconditionally, so counting them as missing would force a Helm + // upgrade every MinForceInterval forever against a render that can + // never produce them. + if g.VeleroNamespace == "" && group == strategyAPIGroup && (kind == "Velero" || kind == "Kubernetes") { continue } key := fmt.Sprintf("%s/%s", kind, name) diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index e287c9872..a07beea7f 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -37,6 +37,7 @@ var gateGVKs = []schema.GroupVersionKind{ {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Kubernetes"}, {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, } @@ -109,6 +110,7 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { ref("Altinity", "cozy-default-altinity"), ref("Velero", "cozy-default-velero-vminstance"), ref("Velero", "cozy-default-velero-vmdisk"), + ref("Kubernetes", "cozy-default-kubernetes"), }, }, } @@ -149,6 +151,7 @@ func allDefaultObjects() []runtime.Object { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), } } @@ -288,8 +291,8 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) { t.Fatal("expected a forced Helm upgrade when default objects are missing") } // 6 strategyRefs (Velero twice under different names) + the BSL. - if len(missing) != 7 { - t.Fatalf("missing = %v, want 7 entries", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want 8 entries", missing) } ann := forceAnnotations(t, dyn) forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] @@ -319,6 +322,7 @@ func TestCheckNoopWhenAllPresent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -454,8 +458,8 @@ func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) { if !forced { t.Fatal("objects release throttled by the earlier credentials force") } - if len(missing) != 7 { - t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want the 7 strategies + the BSL", missing) } if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { t.Errorf("own HelmRelease not forced: %v", ann) @@ -482,7 +486,7 @@ func TestForceSkipsSuspendedHelmRelease(t *testing.T) { if forced { t.Error("forced reported true for a suspended HelmRelease that ignores the annotations") } - if len(missing) != 7 { + if len(missing) != 8 { t.Fatalf("missing = %v, want the objects still reported", missing) } if ann := forceAnnotations(t, dyn); len(ann) != 0 { @@ -599,6 +603,7 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -658,6 +663,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), ) // Rebuild the RESTMapper without BackupStorageLocation: the velero.io API // is not served on this cluster. @@ -1053,8 +1059,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.lastForce = base.Add(-time.Minute) g.now = func() time.Time { return base } - if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 { - t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err) + if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want %d reported, none forced", missing, forced, err, len(allDefaultObjects())) } g.now = func() time.Time { return base.Add(5 * time.Minute) } @@ -1065,8 +1071,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.now = func() time.Time { return base.Add(time.Hour) } missing, forced, err := g.Check(context.Background()) - if err != nil || !forced || len(missing) != 7 { - t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err) + if err != nil || !forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want %d reported and forced again", missing, forced, err, len(allDefaultObjects())) } if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first { t.Errorf("not forced again for unannotated objects: forceAt still %q", first) diff --git a/internal/backupcontroller/kubernetesstrategy_controller.go b/internal/backupcontroller/kubernetesstrategy_controller.go new file mode 100644 index 000000000..1fd47f195 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller.go @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "maps" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + kubernetesAppKind = "Kubernetes" + kubernetesReleasePrefix = "kubernetes-" + + kubernetesLabelMode = "kubernetes.strategy.backups.cozystack.io/mode" + kubernetesModeBackup = "backup" + kubernetesModeRestore = "restore" + kubernetesModeCleanup = "cleanup" + // kubernetesModeVerify checks that the dump object exists before a restore + // changes anything. + kubernetesModeVerify = "verify" + + // kubernetesCleanupAttemptsAnnotation counts failed cleanup Jobs on the + // Backup, so a persistently failing delete (revoked credentials, a removed + // bucket) ends in a released Backup and a Warning rather than a Job storm. + kubernetesCleanupAttemptsAnnotation = "kubernetes.strategy.backups.cozystack.io/cleanup-attempts" + kubernetesCleanupMaxAttempts = 5 + + kubernetesMetaPrefix = "kubernetes.strategy.backups.cozystack.io/" + kubernetesObjectMetaKey = kubernetesMetaPrefix + "object" + kubernetesDatastorePrefixMetaKey = kubernetesMetaPrefix + "datastore-prefix" + kubernetesDatastoreEndpointsMetaKey = kubernetesMetaPrefix + "datastore-endpoints" + kubernetesHelmReleasesMetaKey = kubernetesMetaPrefix + "helmreleases" + kubernetesDataVolumesMetaKey = kubernetesMetaPrefix + "datavolumes" + kubernetesCertWarningsMetaKey = kubernetesMetaPrefix + "certificate-warnings" + kubernetesControlPlaneUIDMetaKey = kubernetesMetaPrefix + "tenantcontrolplane-uid" + kubernetesParamPrefix = kubernetesMetaPrefix + "parameter/" + + // kubernetesCondCertificates is set on the BackupJob once the control-plane + // certificates have been checked; Status=True means at least one expires + // inside the warning window. + kubernetesCondCertificates = "CertificatesExpiring" + + kubernetesPollInterval = 5 * time.Second + // Leaf certificates are rotated by Kamaji and cert-manager, so a near + // expiry is only worth a warning when it is this close. + kubernetesDefaultCertThreshold = 720 * time.Hour + // A CA must outlive every backup that may still be restored, so its + // window is the Velero backup TTL; this is the fallback when none is set. + kubernetesDefaultBackupTTL = 720 * time.Hour + + helmReleaseNameAnnotation = "meta.helm.sh/release-name" + clusterNameLabel = "cluster.x-k8s.io/cluster-name" + helmInstanceLabel = "app.kubernetes.io/instance" + kamajiNameLabel = "kamaji.clastix.io/name" +) + +var ( + tenantControlPlaneGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "tenantcontrolplanes"} + kamajiDataStoreGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "datastores"} + kamajiControlPlaneGVR = schema.GroupVersionResource{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Resource: "kamajicontrolplanes"} + machineDeploymentGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"} + machineGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"} + deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +) + +// kubernetesCertificateSecrets lists, by suffix after the release name, the +// Secrets whose certificates a backup checks. Kamaji and cert-manager rotate +// the leaves; the CAs are irreplaceable, so a restore needs them valid. +var kubernetesCertificateSecrets = []string{ + "ca", + "front-proxy-ca-certificate", + "api-server-certificate", + "api-server-kubelet-client-certificate", + "front-proxy-client-certificate", + "talos-ca", + "talos-tls-cert", +} + +// kubernetesUnlabelledSecrets are issued by cert-manager without the Helm +// instance label the Velero selector relies on; the driver adds it before the +// backup so the Talos PKI is not silently left out. +var kubernetesUnlabelledSecrets = []string{"talos-ca", "talos-tls-cert"} + +func kubernetesReleaseName(appName string) string { + return kubernetesReleasePrefix + appName +} + +func validateKubernetesApplicationRef(ref corev1.TypedLocalObjectReference) error { + if ref.Kind != kubernetesAppKind { + return fmt.Errorf("kubernetes strategy supports applicationRef.kind %s, got %q", kubernetesAppKind, ref.Kind) + } + if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup { + return fmt.Errorf("kubernetes strategy supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) + } + return nil +} + +func kubernetesObjectKey(namespace, appName, backupName string) string { + return fmt.Sprintf("%s/%s/%s.etcd.json", namespace, appName, backupName) +} + +// kubernetesDatastore is where the tenant cluster keeps its state: a key +// prefix in the etcd DataStore shared by every cluster of the tenant. +type kubernetesDatastore struct { + // ControlPlaneUID identifies the cluster across a delete and re-create + // under the same name, which Prefix does not on a Kamaji that keys the + // schema on the namespace and name. + ControlPlaneUID string + Prefix string + Endpoints []string + CertificateSecret string + // SourcePrefix is the prefix the restored dump was taken under. It + // differs from Prefix when a deleted cluster is rebuilt: Kamaji keys a new + // TenantControlPlane's schema on its UID. + SourcePrefix string +} + +func (d kubernetesDatastore) templateContext() map[string]any { + source := d.SourcePrefix + if source == "" { + source = d.Prefix + } + return map[string]any{ + "Prefix": d.Prefix, + "SourcePrefix": source, + "Endpoints": strings.Join(d.Endpoints, ","), + "CertificateSecret": d.CertificateSecret, + } +} + +// etcdEndpointURL gives a scheme to a DataStore endpoint: Kamaji lists them as +// host:port and always talks TLS to an etcd DataStore. +func etcdEndpointURL(endpoint string) string { + if strings.Contains(endpoint, "://") { + return endpoint + } + return "https://" + endpoint +} + +// resolveKubernetesDatastore reads the cluster's key prefix and client +// certificate from its TenantControlPlane and the client URLs from the +// DataStore it points at. +func resolveKubernetesDatastore(ctx context.Context, dyn dynamic.Interface, namespace, appName string) (*kubernetesDatastore, error) { + release := kubernetesReleaseName(appName) + tcp, err := dyn.Resource(tenantControlPlaneGVR).Namespace(namespace).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if driver, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "driver"); driver != "" && driver != "etcd" { + return nil, fmt.Errorf("TenantControlPlane %s/%s uses the %q datastore driver; only etcd is supported", namespace, release, driver) + } + prefix, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + if prefix == "" { + prefix, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStoreSchema") + } + if prefix == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s reports no datastore schema yet", namespace, release) + } + dsName, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "dataStoreName") + if dsName == "" { + dsName, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStore") + } + if dsName == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s names no DataStore", namespace, release) + } + certSecret, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "certificate", "secretName") + if certSecret == "" { + certSecret = release + "-datastore-certificate" + } + ds, err := dyn.Resource(kamajiDataStoreGVR).Get(ctx, dsName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get DataStore %s: %w", dsName, err) + } + raw, _, _ := unstructured.NestedStringSlice(ds.Object, "spec", "endpoints") + if len(raw) == 0 { + return nil, fmt.Errorf("DataStore %s lists no endpoints", dsName) + } + endpoints := make([]string, 0, len(raw)) + for _, e := range raw { + endpoints = append(endpoints, etcdEndpointURL(e)) + } + return &kubernetesDatastore{ControlPlaneUID: string(tcp.GetUID()), Prefix: prefix, Endpoints: endpoints, CertificateSecret: certSecret}, nil +} + +// kubernetesCertificateWarnings reports every certificate in the given Secrets +// that expires inside its window: caWindow for CAs, leafWindow for the rest. +// Output is sorted so a condition message is stable across reconciles. +func kubernetesCertificateWarnings(secrets []corev1.Secret, now time.Time, caWindow, leafWindow time.Duration) []string { + seen := map[string]bool{} + var out []string + for _, s := range secrets { + for key, data := range s.Data { + if !strings.HasSuffix(key, ".crt") { + continue + } + for rest := data; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + window := leafWindow + kind := "certificate" + if cert.IsCA { + window = caWindow + kind = "CA" + } + if !cert.NotAfter.Before(now.Add(window)) { + continue + } + msg := fmt.Sprintf("%s %s/%s (CN=%s) expires %s", kind, s.Name, key, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC3339)) + if !seen[msg] { + seen[msg] = true + out = append(out, msg) + } + } + } + } + sort.Strings(out) + return out +} + +// kubernetesHelmReleaseNames returns the HelmRelease of the cluster followed +// by those of its worker pools. Pools live in separate kubernetes-nodes +// releases; the MachineDeployments of the cluster name them, which also +// covers the older layout where the cluster release owned its pools. +func kubernetesHelmReleaseNames(mds []unstructured.Unstructured, release string) []string { + names := []string{release} + for _, md := range mds { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster != release { + continue + } + if owner := md.GetAnnotations()[helmReleaseNameAnnotation]; owner != "" && !slices.Contains(names, owner) { + names = append(names, owner) + } + } + return names +} + +// sanitizeHelmRelease keeps what is needed to recreate a HelmRelease and +// drops server-set metadata and status. +func sanitizeHelmRelease(hr *unstructured.Unstructured) map[string]any { + labels := map[string]string{} + for k, v := range hr.GetLabels() { + if !strings.HasPrefix(k, "velero.io/") { + labels[k] = v + } + } + meta := map[string]any{ + "name": hr.GetName(), + "namespace": hr.GetNamespace(), + } + if len(labels) > 0 { + meta["labels"] = labels + } + spec, _, _ := unstructured.NestedMap(hr.Object, "spec") + delete(spec, "suspend") + return map[string]any{ + "apiVersion": hr.GetAPIVersion(), + "kind": hr.GetKind(), + "metadata": meta, + "spec": spec, + } +} + +// kubernetesSchemaPattern is every datastore schema Kamaji assigns: a +// TenantControlPlane UID, or _ on an older Kamaji. The Job +// turns a schema into the prefix it runs `etcdctl del --prefix` on in the +// etcd every cluster of the tenant shares, so a value that is empty, holds a +// slash or a dot, or comes from an edited Backup must never reach it. +var kubernetesSchemaPattern = regexp.MustCompile(`^[a-z0-9_-]+$`) + +func validateKubernetesDatastore(ds kubernetesDatastore) error { + if !kubernetesSchemaPattern.MatchString(ds.Prefix) { + return fmt.Errorf("refusing datastore schema %q: want %s", ds.Prefix, kubernetesSchemaPattern) + } + if ds.SourcePrefix != "" && !kubernetesSchemaPattern.MatchString(ds.SourcePrefix) { + return fmt.Errorf("refusing source datastore schema %q: want %s", ds.SourcePrefix, kubernetesSchemaPattern) + } + return nil +} + +func renderKubernetesTemplate( + tmpl corev1.PodTemplateSpec, + appName, namespace, mode, objectKey string, + ds kubernetesDatastore, + parameters map[string]string, +) (*corev1.PodTemplateSpec, error) { + if err := validateKubernetesDatastore(ds); err != nil { + return nil, err + } + return template.Template(&tmpl, map[string]any{ + "Release": map[string]string{ + "Name": appName, + "Namespace": namespace, + "Prefix": kubernetesReleasePrefix, + }, + "Mode": mode, + "ObjectKey": objectKey, + "Datastore": ds.templateContext(), + "Parameters": parameters, + }) +} + +func kubernetesStrategyParameters(b *backupsv1alpha1.Backup) map[string]string { + out := map[string]string{} + for k, v := range b.Spec.DriverMetadata { + if p := strings.TrimPrefix(k, kubernetesParamPrefix); p != k && p != "" { + out[p] = v + } + } + return out +} + +func kubernetesBackupDatastore(b *backupsv1alpha1.Backup) kubernetesDatastore { + ds := kubernetesDatastore{ + Prefix: b.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey], + CertificateSecret: kubernetesReleaseName(b.Spec.ApplicationRef.Name) + "-datastore-certificate", + } + if e := b.Spec.DriverMetadata[kubernetesDatastoreEndpointsMetaKey]; e != "" { + ds.Endpoints = strings.Split(e, ",") + } + return ds +} + +func kubernetesBackupTTL(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.Velero.Spec.TTL.Duration > 0 { + return s.Spec.Velero.Spec.TTL.Duration + } + return kubernetesDefaultBackupTTL +} + +func kubernetesCertThreshold(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.CertificateWarningThreshold != nil && s.Spec.CertificateWarningThreshold.Duration > 0 { + return s.Spec.CertificateWarningThreshold.Duration + } + return kubernetesDefaultCertThreshold +} + +// --------------------------------------------------------------------------- +// BackupJob path +// --------------------------------------------------------------------------- + +// reconcileKubernetes drives a BackupJob for a tenant Kubernetes application in +// three steps, each observed on later reconciles: check the control-plane +// certificates, dump the cluster's etcd prefix with a Job, then back up the +// host-side Secrets and volumes with Velero. The Backup records what a restore +// needs to put both halves back. +func (r *BackupJobReconciler) reconcileKubernetes(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) { + logger := getLogger(ctx) + logger.Debug("reconciling Kubernetes strategy", "backupjob", j.Name, "phase", j.Status.Phase) + + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || + j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(j.Spec.ApplicationRef); err != nil { + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + if j.Status.StartedAt == nil { + fresh := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil { + return ctrl.Result{}, err + } + if fresh.Status.StartedAt == nil { + base := fresh.DeepCopy() + now := metav1.Now() + fresh.Status.StartedAt = &now + fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + j.Status.StartedAt = fresh.Status.StartedAt + j.Status.Phase = fresh.Status.Phase + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef) + if err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name)) + } + return ctrl.Result{}, err + } + + appName := j.Spec.ApplicationRef.Name + release := kubernetesReleaseName(appName) + ds, err := resolveKubernetesDatastore(ctx, r.Interface, j.Namespace, appName) + if err != nil { + if apierrors.IsNotFound(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s has no TenantControlPlane: %v", j.Namespace, appName, err)) + } + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + // A restore that failed partway leaves the control plane stopped on a + // possibly partial prefix, marked with the replica count it had. Backing + // that up would turn the damage into a recovery point, so a run that has + // not dumped anything yet refuses. + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: jobNameForBackupJob(j)}, &batchv1.Job{}); apierrors.IsNotFound(err) { + kcp, kerr := r.Interface.Resource(kamajiControlPlaneGVR).Namespace(j.Namespace).Get(ctx, release, metav1.GetOptions{}) + if kerr != nil && !apierrors.IsNotFound(kerr) && !apimeta.IsNoMatchError(kerr) { + return ctrl.Result{}, kerr + } + if kerr == nil { + if _, stopped := kcp.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; stopped { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s is stopped by an unfinished restore; finish or repair the restore before backing it up", j.Namespace, appName)) + } + } + } else if err != nil { + return ctrl.Result{}, err + } + + if apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates) == nil { + return r.checkKubernetesCertificates(ctx, j, strategy, release) + } + + objectKey := kubernetesObjectKey(j.Namespace, appName, j.Name) + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, appName, j.Namespace, kubernetesModeBackup, objectKey, *ds, resolved.Parameters) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kubernetes strategy: %v", err)) + } + dump, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, jobNameForBackupJob(j), + map[string]string{ + kubernetesLabelMode: kubernetesModeBackup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure etcd dump Job: %v", err)) + } + switch jobConditionState(dump) { + case batchv1.JobFailed: + msg := jobFailureMessage(dump) + if msg == "" { + msg = "etcd dump Job reported Failed" + } + return r.markBackupJobFailed(ctx, j, msg) + case batchv1.JobComplete: + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + veleroBackups := &velerov1.BackupList{} + if err := r.List(ctx, veleroBackups, client.InNamespace(veleroNamespace), client.MatchingLabels{ + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + backupsv1alpha1.OwningJobNameLabel: j.Name, + }); err != nil { + return ctrl.Result{}, err + } + if len(veleroBackups.Items) == 0 { + if err := r.labelKubernetesSecrets(ctx, j.Namespace, release); err != nil { + return ctrl.Result{}, err + } + if err := r.createKubernetesVeleroBackup(ctx, j, strategy, app, resolved); err != nil { + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, fmt.Sprintf("failed to create Velero Backup: %v", err)) + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + if len(veleroBackups.Items) > 1 { + return r.markBackupJobFailed(ctx, j, "more than one Velero Backup references this BackupJob") + } + vb := &veleroBackups.Items[0] + switch vb.Status.Phase { + case velerov1.BackupPhaseCompleted: + case velerov1.BackupPhaseFailed, velerov1.BackupPhasePartiallyFailed, velerov1.BackupPhaseFailedValidation: + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, vb)) + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if j.Status.BackupRef != nil { + return ctrl.Result{}, nil + } + // A failed create is retried rather than failing the run: the dump and + // the Velero backup already exist, and no Backup would own them. + artifact, err := r.createKubernetesBackupArtifact(ctx, j, resolved, vb, *ds, objectKey) + if err != nil { + return ctrl.Result{}, fmt.Errorf("create Backup artifact: %w", err) + } + now := metav1.Now() + j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} + j.Status.CompletedAt = &now + j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "BackupCompleted", + Message: "etcd prefix and Velero backup completed", + }) + return ctrl.Result{}, r.Status().Update(ctx, j) +} + +// failKubernetesBackupAfterDump fails a run whose etcd dump is already in the +// bucket. No Backup will own the dump, so a cleanup-mode Job owned by the +// BackupJob removes it; the dump holds every Secret of the cluster. +func (r *BackupJobReconciler) failKubernetesBackupAfterDump(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, ds kubernetesDatastore, objectKey string, resolved *ResolvedBackupConfig, msg string) (ctrl.Result, error) { + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, j.Spec.ApplicationRef.Name, j.Namespace, kubernetesModeCleanup, objectKey, ds, resolved.Parameters) + if err == nil { + _, err = ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, j.Name+"-discard", + map[string]string{ + kubernetesLabelMode: kubernetesModeCleanup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + } + if err != nil { + msg = fmt.Sprintf("%s; the etcd dump %s could not be scheduled for deletion: %v", msg, objectKey, err) + } + return r.markBackupJobFailed(ctx, j, msg) +} + +// checkKubernetesCertificates records on the BackupJob whether any +// control-plane certificate expires inside its window. It never fails the +// run: an expiring certificate is a reason to act, not to skip the backup. +func (r *BackupJobReconciler) checkKubernetesCertificates(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, release string) (ctrl.Result, error) { + var secrets []corev1.Secret + for _, suffix := range kubernetesCertificateSecrets { + s := corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: release + "-" + suffix}, &s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return ctrl.Result{}, err + } + secrets = append(secrets, s) + } + warnings := kubernetesCertificateWarnings(secrets, time.Now(), kubernetesBackupTTL(strategy), kubernetesCertThreshold(strategy)) + cond := metav1.Condition{ + Type: kubernetesCondCertificates, + Status: metav1.ConditionFalse, + Reason: "CertificatesValid", + Message: "no control-plane certificate expires within the warning window", + } + if len(warnings) > 0 { + cond.Status = metav1.ConditionTrue + cond.Reason = "CertificatesExpiring" + cond.Message = strings.Join(warnings, "; ") + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeWarning, "CertificatesExpiring", cond.Message) + } + } + apimeta.SetStatusCondition(&j.Status.Conditions, cond) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *BackupJobReconciler) labelKubernetesSecrets(ctx context.Context, namespace, release string) error { + for _, suffix := range kubernetesUnlabelledSecrets { + s := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release + "-" + suffix}, s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if s.Labels[helmInstanceLabel] == release { + continue + } + base := s.DeepCopy() + if s.Labels == nil { + s.Labels = map[string]string{} + } + s.Labels[helmInstanceLabel] = release + if err := r.Patch(ctx, s, client.MergeFrom(base)); err != nil { + return fmt.Errorf("label Secret %s/%s: %w", namespace, s.Name, err) + } + } + return nil +} + +func (r *BackupJobReconciler) createKubernetesVeleroBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, app map[string]any, resolved *ResolvedBackupConfig) error { + spec, err := template.Template(&strategy.Spec.Velero.Spec, map[string]any{ + "Application": app, + "Parameters": resolved.Parameters, + }) + if err != nil { + return err + } + // The volumes an in-place restore deletes and replaces are listed before + // Velero starts: each of them is then in the Velero backup unless it was + // deleted first, and a volume created while the backup runs, which Velero + // may have missed, is never deleted by a restore of it. + volumes, err := r.kubernetesDataVolumeNames(ctx, j.Namespace, kubernetesReleaseName(j.Spec.ApplicationRef.Name)) + if err != nil { + return err + } + vb := &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-", j.Namespace, j.Name), + Namespace: veleroNamespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, + Annotations: map[string]string{kubernetesDataVolumesMetaKey: strings.Join(volumes, ",")}, + }, + Spec: *spec, + } + if err := r.Create(ctx, vb); err != nil { + return err + } + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeNormal, "VeleroBackupCreated", fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, vb.Name)) + } + return nil +} + +// kubernetesDataVolumeNames lists the claims backing the cluster's +// PersistentVolumes: the DataVolumes and PVCs the CSI driver labels with the +// cluster name. A PVC is listed even without a DataVolume, so an in-place +// restore clears it too rather than leaving Velero to skip it. +func (r *BackupJobReconciler) kubernetesDataVolumeNames(ctx context.Context, ns, release string) ([]string, error) { + selector := clusterNameLabel + "=" + release + seen := map[string]bool{} + dvs, err := r.Interface.Resource(dataVolumeGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + for _, dv := range dvs.Items { + seen[dv.GetName()] = true + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := r.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{clusterNameLabel: release}); err != nil { + return nil, err + } + for _, pvc := range pvcs.Items { + seen[pvc.Name] = true + } + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// createKubernetesBackupArtifact records, besides the Velero backup and the +// etcd object, the cluster's HelmReleases and data DataVolumes. The +// HelmReleases are kept here rather than in Velero because selecting them by +// application label would also select the worker system disks, which the +// lineage webhook labels after the same application. +func (r *BackupJobReconciler) createKubernetesBackupArtifact( + ctx context.Context, + j *backupsv1alpha1.BackupJob, + resolved *ResolvedBackupConfig, + vb *velerov1.Backup, + ds kubernetesDatastore, + objectKey string, +) (*backupsv1alpha1.Backup, error) { + release := kubernetesReleaseName(j.Spec.ApplicationRef.Name) + mds, err := r.Interface.Resource(machineDeploymentGVR).Namespace(j.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil && !apierrors.IsNotFound(err) && !apimeta.IsNoMatchError(err) { + return nil, err + } + var mdItems []unstructured.Unstructured + if mds != nil { + mdItems = mds.Items + } + var releases []map[string]any + for _, name := range kubernetesHelmReleaseNames(mdItems, release) { + hr, err := r.Interface.Resource(helmReleaseGVR).Namespace(j.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + releases = append(releases, sanitizeHelmRelease(hr)) + } + releasesJSON, err := json.Marshal(releases) + if err != nil { + return nil, err + } + + driverMD := map[string]string{ + veleroBackupNameMetadataKey: vb.Name, + veleroBackupNamespaceMetadataKey: vb.Namespace, + kubernetesObjectMetaKey: objectKey, + kubernetesDatastorePrefixMetaKey: ds.Prefix, + kubernetesDatastoreEndpointsMetaKey: strings.Join(ds.Endpoints, ","), + kubernetesHelmReleasesMetaKey: string(releasesJSON), + kubernetesDataVolumesMetaKey: vb.Annotations[kubernetesDataVolumesMetaKey], + kubernetesControlPlaneUIDMetaKey: ds.ControlPlaneUID, + } + if c := apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates); c != nil && c.Status == metav1.ConditionTrue { + driverMD[kubernetesCertWarningsMetaKey] = c.Message + } + for k, v := range resolved.Parameters { + driverMD[kubernetesParamPrefix+k] = v + } + + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: j.Name, Namespace: j.Namespace}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: j.Spec.ApplicationRef, + StrategyRef: resolved.StrategyRef, + TakenAt: metav1.Now(), + DriverMetadata: driverMD, + PlanRef: j.Spec.PlanRef, + }, + Status: backupsv1alpha1.BackupStatus{ + Phase: backupsv1alpha1.BackupPhaseReady, + Artifact: &backupsv1alpha1.BackupArtifact{URI: fmt.Sprintf("velero://%s/%s", vb.Namespace, vb.Name)}, + }, + } + if err := r.Create(ctx, backup); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); err != nil { + return nil, err + } + return existing, nil + } + return backup, nil +} + +// --------------------------------------------------------------------------- +// Backup deletion cleanup +// --------------------------------------------------------------------------- + +// cleanupKubernetesBackup removes both halves of a deleted Backup: the Velero +// backup through a DeleteBackupRequest, and the etcd dump object through a +// cleanup-mode Job, waiting for the Job before the Backup is released. The +// escape hatches match the Redis driver's. +func (r *BackupReconciler) cleanupKubernetesBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if err := r.cleanupVeleroBackup(ctx, backup); err != nil { + return ctrl.Result{}, err + } + + jobName := backup.Name + "-cleanup" + if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" { + existing := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil { + _ = r.deleteRedisCleanupJob(ctx, existing) + } + return ctrl.Result{}, nil + } + objectKey := backup.Spec.DriverMetadata[kubernetesObjectMetaKey] + if objectKey == "" { + return ctrl.Result{}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.releaseRedisCleanup(ctx, backup, objectKey, "strategy CR is gone"), nil + } + return ctrl.Result{}, err + } + + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job) + switch { + case apierrors.IsNotFound(err): + terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace) + if nsErr != nil { + return ctrl.Result{}, nsErr + } + if terminating { + return r.releaseRedisCleanup(ctx, backup, objectKey, "namespace is terminating"), nil + } + if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot project credentials: %v", perr)), nil + } + rendered, rerr := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, backup.Namespace, + kubernetesModeCleanup, objectKey, kubernetesBackupDatastore(backup), kubernetesStrategyParameters(backup)) + if rerr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil + } + cleanup := buildRedisCleanupJob(backup.Namespace, jobName, rendered) + cleanup.Labels = map[string]string{kubernetesLabelMode: kubernetesModeCleanup} + delete(cleanup.Spec.Template.Labels, redisLabelMode) + maps.Copy(cleanup.Spec.Template.Labels, cleanup.Labels) + if cerr := r.Create(ctx, cleanup); cerr != nil && !apierrors.IsAlreadyExists(cerr) { + if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil + } + return ctrl.Result{}, cerr + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + case err != nil: + return ctrl.Result{}, err + } + + if !job.DeletionTimestamp.IsZero() { + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + switch jobConditionState(job) { + case batchv1.JobComplete: + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{}, nil + case batchv1.JobFailed: + attempts, _ := strconv.Atoi(backup.Annotations[kubernetesCleanupAttemptsAnnotation]) + attempts++ + if attempts >= kubernetesCleanupMaxAttempts { + _ = r.deleteRedisCleanupJob(ctx, job) + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("the cleanup Job failed %d times: %s", attempts, jobFailureMessage(job))), nil + } + base := backup.DeepCopy() + if backup.Annotations == nil { + backup.Annotations = map[string]string{} + } + backup.Annotations[kubernetesCleanupAttemptsAnnotation] = strconv.Itoa(attempts) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{RequeueAfter: time.Duration(attempts) * 30 * time.Second}, nil + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_controller_test.go b/internal/backupcontroller/kubernetesstrategy_controller_test.go new file mode 100644 index 000000000..590f82ae9 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller_test.go @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "strconv" + "strings" + "testing" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +const ( + k8sTestNS = "tenant-a" + k8sTestApp = "c1" + k8sTestRelease = "kubernetes-c1" +) + +var kubernetesAppGVR = schema.GroupVersionResource{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Resource: "kubernetes"} + +func testCertPEM(t *testing.T, isCA bool, notAfter time.Time) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test"}, + NotBefore: notAfter.Add(-time.Hour), + NotAfter: notAfter, + IsCA: isCA, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func unstr(gvk schema.GroupVersionKind, ns, name string, fields map[string]any) *unstructured.Unstructured { + u := &unstructured.Unstructured{Object: map[string]any{}} + for k, v := range fields { + u.Object[k] = v + } + u.SetGroupVersionKind(gvk) + u.SetNamespace(ns) + u.SetName(name) + return u +} + +func TestKubernetesCertificateWarnings(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + secrets := []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, now.Add(10*24*time.Hour)), + "ca.key": []byte("not a certificate"), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-soon"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(24*time.Hour)), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-later"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(10*24*time.Hour)), + }}, + } + + got := kubernetesCertificateWarnings(secrets, now, 30*24*time.Hour, 2*24*time.Hour) + if len(got) != 2 { + t.Fatalf("got %d warnings %v, want 2", len(got), got) + } + if !strings.HasPrefix(got[0], "CA ca/ca.crt") || !strings.HasPrefix(got[1], "certificate leaf-soon/tls.crt") { + t.Fatalf("unexpected warnings %v", got) + } + + if got := kubernetesCertificateWarnings(secrets, now, time.Hour, time.Hour); len(got) != 0 { + t.Fatalf("want no warnings inside a short window, got %v", got) + } +} + +func TestKubernetesHelmReleaseNames(t *testing.T) { + mdGVK := schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"} + md := func(name, cluster, owner string) unstructured.Unstructured { + u := unstr(mdGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"clusterName": cluster}}) + u.SetAnnotations(map[string]string{helmReleaseNameAnnotation: owner}) + return *u + } + got := kubernetesHelmReleaseNames([]unstructured.Unstructured{ + md("a", k8sTestRelease, "kubernetes-nodes-c1-a"), + md("b", k8sTestRelease, "kubernetes-nodes-c1-b"), + md("legacy", k8sTestRelease, k8sTestRelease), + md("other", "kubernetes-c2", "kubernetes-nodes-c2-a"), + }, k8sTestRelease) + want := []string{k8sTestRelease, "kubernetes-nodes-c1-a", "kubernetes-nodes-c1-b"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestParseKubernetesRestoreOptions(t *testing.T) { + o, err := parseKubernetesRestoreOptions(nil) + if err != nil || !o.restoreEtcd() || !o.restoreVolumes() { + t.Fatalf("defaults: %+v %v", o, err) + } + o, err = parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"volumes":false}`)}) + if err != nil || !o.restoreEtcd() || o.restoreVolumes() { + t.Fatalf("etcd only: %+v %v", o, err) + } + if _, err := parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"etcd":false,"volumes":false}`)}); err == nil { + t.Fatal("want an error when both halves are disabled") + } +} + +func TestKubernetesVeleroRestoreSpecPhases(t *testing.T) { + strategy := &strategyv1alpha1.Kubernetes{} + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS}, + Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: k8sTestApp}}, + } + secrets, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseSecrets, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if strings.Join(secrets.IncludedResources, ",") != "secrets" || len(secrets.OrLabelSelectors) != 2 { + t.Fatalf("secrets phase: %+v", secrets) + } + volumes, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseVolumes, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if volumes.OrLabelSelectors[0].MatchLabels[clusterNameLabel] != k8sTestRelease || volumes.RestorePVs == nil || !*volumes.RestorePVs { + t.Fatalf("volumes phase: %+v", volumes) + } + if _, err := kubernetesVeleroRestoreSpec(strategy, backup, "bogus", k8sTestRelease); err == nil { + t.Fatal("want an error for an unknown phase") + } +} + +type kubernetesTestEnv struct { + backup *BackupJobReconciler + restore *RestoreJobReconciler + client client.Client +} + +// newKubernetesTestEnv seeds a live tenant cluster: the application, its +// HelmReleases, TenantControlPlane, DataStore, KamajiControlPlane, one worker +// pool and one data DataVolume. +const k8sTestControlPlaneUID = "6b0c4f1e-2d3a-4e5f-8a9b-0c1d2e3f4a5b" + +func newKubernetesTestEnv(t *testing.T, objs ...client.Object) *kubernetesTestEnv { + t.Helper() + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = backupsv1alpha1.AddToScheme(s) + _ = strategyv1alpha1.AddToScheme(s) + _ = velerov1.AddToScheme(s) + + listKinds := map[schema.GroupVersionResource]string{ + kubernetesAppGVR: "KubernetesList", + helmReleaseGVR: "HelmReleaseList", + tenantControlPlaneGVR: "TenantControlPlaneList", + kamajiDataStoreGVR: "DataStoreList", + kamajiControlPlaneGVR: "KamajiControlPlaneList", + machineDeploymentGVR: "MachineDeploymentList", + machineGVR: "MachineList", + dataVolumeGVR: "DataVolumeList", + deploymentGVR: "DeploymentList", + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(s, listKinds) + seed := func(gvr schema.GroupVersionResource, u *unstructured.Unstructured) { + if err := dyn.Tracker().Create(gvr, u, u.GetNamespace()); err != nil { + t.Fatalf("seed %s: %v", u.GetName(), err) + } + } + appGVK := schema.GroupVersionKind{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Kind: kubernetesAppKind} + seed(kubernetesAppGVR, unstr(appGVK, k8sTestNS, k8sTestApp, map[string]any{"spec": map[string]any{}})) + hrGVK := schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"} + for _, name := range []string{k8sTestRelease, "kubernetes-nodes-c1-md0"} { + hr := unstr(hrGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"interval": "5m"}}) + hr.SetLabels(map[string]string{"apps.cozystack.io/application.name": name, "velero.io/backup-name": "old"}) + seed(helmReleaseGVR, hr) + } + tcp := unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "status": map[string]any{"storage": map[string]any{ + "driver": "etcd", + "dataStoreName": "tenant-a", + "setup": map[string]any{"schema": "tenant_a_kubernetes_c1"}, + "certificate": map[string]any{"secretName": k8sTestRelease + "-datastore-certificate"}, + }}, + }) + tcp.SetUID(k8sTestControlPlaneUID) + seed(tenantControlPlaneGVR, tcp) + seed(kamajiDataStoreGVR, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "DataStore"}, "", "tenant-a", map[string]any{ + "spec": map[string]any{"endpoints": []any{"etcd.tenant-a.svc:2379"}}, + })) + seed(kamajiControlPlaneGVR, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + })) + md := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"}, k8sTestNS, k8sTestRelease+"-md0", map[string]any{ + "spec": map[string]any{"clusterName": k8sTestRelease, "replicas": int64(3)}, + }) + md.SetAnnotations(map[string]string{helmReleaseNameAnnotation: "kubernetes-nodes-c1-md0"}) + seed(machineDeploymentGVR, md) + dv := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-1234", nil) + dv.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + seed(dataVolumeGVR, dv) + seed(deploymentGVR, unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease+"-cluster-autoscaler", map[string]any{ + "spec": map[string]any{"replicas": int64(1)}, + })) + + base := []client.Object{ + &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "tenant-a.bj1-xyz"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseCompleted}, + }, + &strategyv1alpha1.Kubernetes{ + ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-kubernetes"}, + Spec: strategyv1alpha1.KubernetesSpec{ + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "etcd", + Image: "img", + Args: []string{"{{ .Mode }} {{ .ObjectKey }} {{ .Datastore.Prefix }} {{ .Datastore.Endpoints }} {{ .Datastore.CertificateSecret }} {{ .Datastore.SourcePrefix }}"}, + }}}}, + Velero: strategyv1alpha1.VeleroTemplate{Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"{{ .Application.metadata.namespace }}"}, + TTL: metav1.Duration{Duration: 720 * time.Hour}, + }}, + }, + }, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, time.Now().Add(24*time.Hour)), + }}, + } + c := clientfake.NewClientBuilder().WithScheme(s). + WithObjects(append(base, objs...)...). + WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). + Build() + mapper := &mockRESTMapper{mapping: &meta.RESTMapping{Resource: kubernetesAppGVR, GroupVersionKind: appGVK, Scope: meta.RESTScopeNamespace}} + return &kubernetesTestEnv{ + backup: &BackupJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + restore: &RestoreJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + client: c, + } +} + +func k8sResolved() *ResolvedBackupConfig { + return &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new("strategy.backups.cozystack.io"), Kind: strategyv1alpha1.KubernetesStrategyKind, Name: "cozy-default-kubernetes"}, + Parameters: map[string]string{"p": "v"}, + } +} + +func TestReconcileKubernetesBackup(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + step := func() { + t.Helper() + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + + step() // StartedAt + step() // certificates + cond := meta.FindStatusCondition(get().Status.Conditions, kubernetesCondCertificates) + if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, k8sTestRelease+"-ca/ca.crt") { + t.Fatalf("certificate condition: %+v", cond) + } + + step() // dump Job + job := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1-backup"}, job); err != nil { + t.Fatalf("dump Job: %v", err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + want := "backup tenant-a/c1/bj1.etcd.json tenant_a_kubernetes_c1 https://etcd.tenant-a.svc:2379 kubernetes-c1-datastore-certificate tenant_a_kubernetes_c1" + if args != want { + t.Fatalf("rendered args %q, want %q", args, want) + } + step() + vbs := &velerov1.BackupList{} + _ = env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}) + if len(vbs.Items) != 0 { + t.Fatal("Velero Backup created before the dump Job completed") + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, job); err != nil { + t.Fatal(err) + } + // A claim the CSI driver created without a DataVolume is still one an + // in-place restore has to clear before Velero writes it back. + if err := env.client.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: k8sTestNS, Name: "pvc-plain", Labels: map[string]string{clusterNameLabel: k8sTestRelease}, + }}); err != nil { + t.Fatal(err) + } + step() + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + // A volume created while Velero runs may be missing from its backup, so a + // restore of this backup must never delete it. + late := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-late", nil) + late.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := env.backup.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Create(ctx, late, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if ns := vbs.Items[0].Spec.IncludedNamespaces; len(ns) != 1 || ns[0] != k8sTestNS { + t.Fatalf("Velero spec not rendered: %v", ns) + } + talosCA := &corev1.Secret{} + _ = env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}, talosCA) + if talosCA.Labels[helmInstanceLabel] != k8sTestRelease { + t.Fatalf("talos CA not labelled for the Velero selector: %v", talosCA.Labels) + } + + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseCompleted + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + // A pool that is suspended while the backup runs must not come back + // suspended when the cluster is rebuilt from it. + if _, err := env.backup.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(ctx, "kubernetes-nodes-c1-md0", types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + step() + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || got.Status.BackupRef == nil { + t.Fatalf("BackupJob not succeeded: %+v", got.Status) + } + backup := &backupsv1alpha1.Backup{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1"}, backup); err != nil { + t.Fatal(err) + } + md := backup.Spec.DriverMetadata + if md[veleroBackupNameMetadataKey] != vb.Name || md[kubernetesDataVolumesMetaKey] != "pvc-1234,pvc-plain" || + md[kubernetesDatastorePrefixMetaKey] != "tenant_a_kubernetes_c1" || md[kubernetesParamPrefix+"p"] != "v" || + md[kubernetesCertWarningsMetaKey] == "" || md[kubernetesControlPlaneUIDMetaKey] != k8sTestControlPlaneUID { + t.Fatalf("driver metadata: %v", md) + } + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil || len(releases) != 2 { + t.Fatalf("recorded HelmReleases: %v", releases) + } + raw, _ := json.Marshal(releases) + if strings.Contains(string(raw), "velero.io/") || strings.Contains(string(raw), "suspend") { + t.Fatalf("Velero labels or the suspend flag leaked into the recorded HelmReleases: %s", raw) + } +} + +func k8sBackupForRestore() *backupsv1alpha1.Backup { + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp}, + StrategyRef: k8sResolved().StrategyRef, + DriverMetadata: map[string]string{ + veleroBackupNameMetadataKey: "tenant-a.bj1-xyz", + kubernetesObjectMetaKey: "tenant-a/c1/bj1.etcd.json", + kubernetesDatastorePrefixMetaKey: "tenant_a_kubernetes_c1", + kubernetesControlPlaneUIDMetaKey: k8sTestControlPlaneUID, + kubernetesDatastoreEndpointsMetaKey: "https://etcd.tenant-a.svc:2379", + kubernetesDataVolumesMetaKey: "pvc-1234", + kubernetesHelmReleasesMetaKey: `[{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-c1"},"spec":{"interval":"5m"}}]`, + }, + }, + } +} + +// k8sRestoreRun drives one RestoreJob through the test environment. +type k8sRestoreRun struct { + t *testing.T + env *kubernetesTestEnv + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup +} + +func newK8sRestoreRun(t *testing.T, options string, backup *backupsv1alpha1.Backup) *k8sRestoreRun { + t.Helper() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + } + if options != "" { + rj.Spec.Options = &runtime.RawExtension{Raw: []byte(options)} + } + return &k8sRestoreRun{t: t, env: newKubernetesTestEnv(t, backup, rj), rj: rj, backup: backup} +} + +func (r *k8sRestoreRun) get() *backupsv1alpha1.RestoreJob { + r.t.Helper() + out := &backupsv1alpha1.RestoreJob{} + if err := r.env.client.Get(context.Background(), client.ObjectKeyFromObject(r.rj), out); err != nil { + r.t.Fatal(err) + } + return out +} + +func (r *k8sRestoreRun) step() { + r.t.Helper() + if _, err := r.env.restore.reconcileKubernetesRestore(context.Background(), r.get(), r.backup); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishJob(name string, condType batchv1.JobConditionType) { + r.t.Helper() + job := &batchv1.Job{} + if err := r.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: name}, job); err != nil { + r.t.Fatalf("Job %s: %v", name, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: condType, Status: corev1.ConditionTrue, Message: "boom"}} + if err := r.env.client.Status().Update(context.Background(), job); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishVeleroRestore(phase velerov1.RestorePhase) { + r.t.Helper() + list := &velerov1.RestoreList{} + if err := r.env.client.List(context.Background(), list); err != nil || len(list.Items) == 0 { + r.t.Fatalf("Velero Restores: %v %v", len(list.Items), err) + } + vr := list.Items[len(list.Items)-1] + vr.Status.Phase = phase + if err := r.env.client.Update(context.Background(), &vr); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) obj(gvr schema.GroupVersionResource, name string) *unstructured.Unstructured { + r.t.Helper() + u, err := r.env.restore.Interface.Resource(gvr).Namespace(k8sTestNS).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + r.t.Fatalf("get %s %s: %v", gvr.Resource, name, err) + } + return u +} + +func (r *k8sRestoreRun) replicas(gvr schema.GroupVersionResource, name string) (int64, bool) { + n, found, _ := unstructured.NestedInt64(r.obj(gvr, name).Object, "spec", "replicas") + return n, found +} + +func (r *k8sRestoreRun) suspended(name string) bool { + s, _, _ := unstructured.NestedBool(r.obj(helmReleaseGVR, name).Object, "spec", "suspend") + return s +} + +func (r *k8sRestoreRun) addMachine(name string) { + r.t.Helper() + m := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "Machine"}, k8sTestNS, name, nil) + m.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := r.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Create(context.Background(), m, metav1.CreateOptions{}); err != nil { + r.t.Fatal(err) + } +} + +// verifyAndCapture runs the preflight and the mode decision. +func (r *k8sRestoreRun) verifyAndCapture(etcd bool) { + r.t.Helper() + r.step() // StartedAt + r.step() // preflight + if etcd { + r.finishJob("rj-verify", batchv1.JobComplete) + r.step() + } + if !meta.IsStatusConditionTrue(r.get().Status.Conditions, kubernetesCondVerified) { + r.t.Fatalf("backup not verified: %+v", r.get().Status) + } + r.step() // capture +} + +func TestReconcileKubernetesRestoreInPlaceDefault(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + + run.step() // stop: a worker is still there + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatal("control plane scaled down while a worker Machine still needs it to delete its Node") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 0 { + t.Fatalf("worker pool not scaled down: %d", n) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not scaled down: %d", n) + } + if _, ok := run.obj(machineGVR, "w-1").GetAnnotations()[machineExcludeDrainAnnotation]; !ok { + t.Fatal("Machine not released from draining") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondStopped); c == nil || c.Status != metav1.ConditionFalse || !strings.Contains(c.Message, "Machine") { + t.Fatalf("stop progress not reported: %+v", c) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted while a worker may still mount it") + } + + if err := run.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Delete(context.Background(), "w-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + run.step() // stop settles + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err == nil { + t.Fatal("backed-up data DataVolume not deleted once the workers were gone") + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane not scaled down after the workers were gone") + } + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped") + } + + run.step() // etcd Job + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() // etcd done + run.step() // volumes Velero Restore + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() // volumes done + run.step() // resume + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + for _, c := range []struct { + gvr schema.GroupVersionResource + name string + want int64 + }{ + {kamajiControlPlaneGVR, k8sTestRelease, 2}, + {machineDeploymentGVR, k8sTestRelease + "-md0", 3}, + {deploymentGVR, k8sTestRelease + "-cluster-autoscaler", 1}, + } { + if n, _ := run.replicas(c.gvr, c.name); n != c.want { + t.Errorf("%s %s replicas = %d, want %d", c.gvr.Resource, c.name, n, c.want) + } + if _, left := run.obj(c.gvr, c.name).GetAnnotations()[kubernetesReplicasBeforeAnnotation]; left { + t.Errorf("%s %s keeps the restore's replica record", c.gvr.Resource, c.name) + } + } + if run.suspended(k8sTestRelease) || run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("HelmRelease left suspended") + } +} + +func TestReconcileKubernetesRestoreInPlaceEtcdOnly(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not scaled down: %d", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("workers touched by an etcd-only restore: %d", n) + } + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane replicas not restored: %d", n) + } + list := &velerov1.RestoreList{} + _ = run.env.client.List(context.Background(), list) + if len(list.Items) != 0 { + t.Fatal("an etcd-only restore created a Velero Restore") + } +} + +func TestReconcileKubernetesRestoreKeepsPreSuspendedRelease(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + patch := []byte(`{"spec":{"suspend":true}}`) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(context.Background(), "kubernetes-nodes-c1-md0", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if run.get().Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", run.get().Status) + } + if !run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("a HelmRelease suspended before the restore was resumed by it") + } + if run.suspended(k8sTestRelease) { + t.Fatal("the cluster HelmRelease the restore suspended was left suspended") + } +} + +func TestReconcileKubernetesRestoreEtcdFailureLeavesControlPlaneStopped(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() + run.step() + run.addJobPod("rj-restore", kubernetesJobWriting) + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("want a failed restore with the etcd write marked failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought back up on a possibly partial etcd prefix") + } + if !strings.Contains(got.Status.Message+fmtConditions(got), ".pre-restore") { + t.Fatalf("failure does not point at the pre-restore copy: %+v", got.Status) + } + + run.env.restore.cleanupKubernetesRestore(context.Background(), got) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up on a possibly partial prefix") + } +} + +func fmtConditions(rj *backupsv1alpha1.RestoreJob) string { + var b strings.Builder + for _, c := range rj.Status.Conditions { + b.WriteString(c.Message) + } + return b.String() +} + +func TestCleanupKubernetesRestoreResumesStoppedCluster(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop: workers gone, volumes deleted, control plane stopped + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not stopped: %d", n) + } + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("deleting the RestoreJob left the control plane at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("deleting the RestoreJob left the worker pool at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("deleting the RestoreJob left the HelmRelease suspended") + } +} + +func TestReconcileKubernetesRestoreStopDeadline(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // stop starts, a worker remains + rj := run.get() + cond := meta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + cond.LastTransitionTime = metav1.NewTime(time.Now().Add(-restorePreparationDeadline - time.Minute)) + if err := run.env.client.Status().Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a stop that never settles must fail the restore: %+v", got.Status) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool not resumed after the stop timed out: %d", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("HelmRelease left suspended after the stop timed out") + } +} + +func TestReconcileKubernetesRestoreVerifiesBeforeChangingAnything(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "expired" + run := newK8sRestoreRun(t, "", backup) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing Velero backup must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the backup was verified") + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted before the backup was verified") + } + + backup = k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "partial" + run = newK8sRestoreRun(t, "", backup) + if err := run.env.client.Create(context.Background(), &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "partial"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhasePartiallyFailed}, + }); err != nil { + t.Fatal(err) + } + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a Velero backup that did not complete must fail the restore: %+v", got.Status) + } + + run = newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.step() + run.step() + run.finishJob("rj-verify", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing etcd dump must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the dump was verified") + } +} + +func k8sBackupOfDeletedCluster() *backupsv1alpha1.Backup { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = `[` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-gone"},"spec":{"interval":"5m"}},` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-nodes-gone-md0"},"spec":{"interval":"5m"}}]` + return backup +} + +// TestReconcileKubernetesRestoreRecreate rebuilds a deleted cluster whose new +// TenantControlPlane gets a fresh, UID-keyed schema: the dump must be written +// under that schema, and the worker pools come back only after it is. +func TestReconcileKubernetesRestoreRecreate(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeRecreate || state.Release != "kubernetes-gone" { + t.Fatalf("want recreate mode, got %+v", state) + } + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + + run.step() // cluster HelmRelease, waiting for the TenantControlPlane + dyn := run.env.restore.Interface + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-gone", metav1.GetOptions{}); err != nil { + t.Fatalf("cluster HelmRelease not recreated: %v", err) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err == nil { + t.Fatal("worker pool recreated before the etcd state was written back") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondRecreated); c == nil || c.Status != metav1.ConditionFalse { + t.Fatalf("wait for the new control plane not reported: %+v", c) + } + run.step() // a later poll, the control plane still coming up + if ann := run.obj(helmReleaseGVR, "kubernetes-gone").GetAnnotations(); ann["reconcile.fluxcd.io/forceAt"] != "" { + t.Fatalf("the cluster HelmRelease the rebuild created is forced through a Helm upgrade on a poll: %v", ann) + } + + const newSchema = "0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10" + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": newSchema}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stop it for the etcd write + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 0 { + t.Fatalf("new control plane not stopped before the etcd write: %d", n) + } + run.step() // etcd Job + job := &batchv1.Job{} + if err := run.env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err != nil { + t.Fatal(err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + if !strings.Contains(args, " "+newSchema+" ") || !strings.HasSuffix(args, " tenant_a_kubernetes_c1") || !strings.Contains(args, "kubernetes-gone-datastore-certificate") { + t.Fatalf("restore Job does not write the dump under the new cluster's schema: %q", args) + } + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() // resume, worker pools + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("new control plane not started again: %d", n) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err != nil { + t.Fatalf("worker pool not recreated: %v", err) + } +} + +func TestReconcileKubernetesRestoreRecreateSecretsFailure(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets restore + list := &velerov1.RestoreList{} + if err := run.env.client.List(context.Background(), list); err != nil || len(list.Items) != 1 { + t.Fatalf("Velero Restores: %d %v", len(list.Items), err) + } + vr := list.Items[0] + if vr.Labels[kubernetesRestorePhaseLabel] != kubernetesPhaseSecrets || vr.Spec.BackupName != "tenant-a.bj1-xyz" || vr.Spec.ResourceModifier == nil { + t.Fatalf("secrets Restore: %+v", vr) + } + cm := &corev1.ConfigMap{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: veleroNamespace, Name: vr.Spec.ResourceModifier.Name}, cm); err != nil { + t.Fatal(err) + } + if !strings.Contains(cm.Data["resource-modifier-rules.yaml"], "ownerReferences: null") { + t.Fatalf("modifiers do not strip ownerReferences: %s", cm.Data["resource-modifier-rules.yaml"]) + } + run.finishVeleroRestore(velerov1.RestorePhaseFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a failed Velero Restore must fail the RestoreJob: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRecreateRejectsBadReleaseRecord(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = "not json" + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("an unreadable HelmRelease record must fail the rebuild: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRejectsRename(t *testing.T) { + backup := k8sBackupForRestore() + run := newK8sRestoreRun(t, "", backup) + rj := run.get() + rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: "other"} + if _, err := run.env.restore.reconcileKubernetesRestore(context.Background(), rj, backup); err != nil { + t.Fatal(err) + } + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore into another application must fail: %+v", got.Status) + } +} + +func TestResolveKubernetesDatastoreRejectsNonEtcd(t *testing.T) { + env := newKubernetesTestEnv(t) + dyn := env.backup.Interface + tcp, _ := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(context.Background(), k8sTestRelease, metav1.GetOptions{}) + _ = unstructured.SetNestedField(tcp.Object, "PostgreSQL", "status", "storage", "driver") + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(context.Background(), tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := resolveKubernetesDatastore(context.Background(), dyn, k8sTestNS, k8sTestApp); err == nil { + t.Fatal("want an error for a non-etcd datastore") + } +} + +func TestCleanupKubernetesBackupGivesUpAfterRepeatedFailures(t *testing.T) { + ctx := context.Background() + backup := k8sBackupForRestore() + backup.Annotations = map[string]string{kubernetesCleanupAttemptsAnnotation: strconv.Itoa(kubernetesCleanupMaxAttempts - 1)} + failed := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1-cleanup"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, + } + env := newKubernetesTestEnv(t, backup, failed) + br := &BackupReconciler{Client: env.client, Scheme: env.backup.Scheme, Recorder: record.NewFakeRecorder(10)} + res, err := br.cleanupKubernetesBackup(ctx, backup) + if err != nil { + t.Fatal(err) + } + if res.RequeueAfter != 0 { + t.Fatalf("cleanup keeps retrying after %d failures: %+v", kubernetesCleanupMaxAttempts, res) + } +} + +func (r *k8sRestoreRun) addJobPod(job, message string) { + r.t.Helper() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: job + "-pod", Labels: map[string]string{"job-name": job}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: message + "\n"}}, + }}}, + } + if err := r.env.client.Create(context.Background(), pod); err != nil { + r.t.Fatal(err) + } +} + +func TestReconcileKubernetesRestoreFailureBeforeWriteResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("a Job that never touched the prefix must fail without marking it partial: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped although the prefix was never changed: %d", n) + } +} + +func TestCleanupKubernetesRestoreKeepsClusterStoppedWhileRestoreJobRuns(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job, still running + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up while the restore Job may be rewriting the prefix") + } + job := &batchv1.Job{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err == nil { + t.Fatal("the running restore Job was left to keep writing") + } +} + +func TestReconcileKubernetesRestoreIgnoresModeAnnotation(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + rj := run.get() + rj.Annotations = map[string]string{"kubernetes.strategy.backups.cozystack.io/restore-state": `{"mode":"recreate","release":"kubernetes-c1"}`} + if err := run.env.client.Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeInPlace { + t.Fatalf("a live cluster must be restored in place whatever the RestoreJob claims: %+v", state) + } +} + +func TestReconcileKubernetesBackupDiscardsDumpWhenVeleroFails(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj2"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + step := func() { + t.Helper() + got := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), got); err != nil { + t.Fatal(err) + } + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + step() + step() + step() + dump := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-backup"}, dump); err != nil { + t.Fatal(err) + } + dump.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, dump); err != nil { + t.Fatal(err) + } + step() + vbs := &velerov1.BackupList{} + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj2"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseFailed + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + step() + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob not failed: %+v", got.Status) + } + discard := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-discard"}, discard); err != nil { + t.Fatalf("the orphaned etcd dump is not scheduled for deletion: %v", err) + } + if args := discard.Spec.Template.Spec.Containers[0].Args[0]; !strings.HasPrefix(args, "cleanup tenant-a/c1/bj2.etcd.json") { + t.Fatalf("discard Job does not delete this run's dump: %q", args) + } +} + +func TestReconcileKubernetesRestoreRecreateFailureBeforeWriteStartsControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": "uid-1"}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("rebuild not failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("the rebuilt control plane is left stopped although its prefix was never changed: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobThatNeverRanResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() // still waiting on the Job + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a container that never ran was taken for a started Job") + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped after a restore Job that never ran: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobPodsGoneAfterStart(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a running restore Job was not recorded as started") + } + // The Job deadline deletes its Pods and fails it. + if err := run.env.client.Delete(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought up after a restore Job that ran and left no trace of how far it got") + } +} + +func TestReconcileKubernetesRestoreRetriesStopAfterAPIError(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + failed := false + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("patch", "kamajicontrolplanes", func(clienttesting.Action) (bool, runtime.Object, error) { + if failed { + return false, nil, nil + } + failed = true + return true, nil, fmt.Errorf("etcdserver: request timed out") + }) + run.step() + got := run.get() + if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("one API error aborted the restore: %+v", got.Status) + } + if c := meta.FindStatusCondition(got.Status.Conditions, kubernetesCondStopped); c == nil || !strings.Contains(c.Message, "API error") { + t.Fatalf("the retried API error is not reported: %+v", c) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("the stop did not go through once the API answered again") + } +} + +// driveRecreateToEtcdJob runs a rebuild up to the etcd restore Job, the new +// TenantControlPlane reporting schema. +func driveRecreateToEtcdJob(t *testing.T, run *k8sRestoreRun, schemaName string) { + t.Helper() + ctx := context.Background() + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": schemaName}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job +} + +func TestReconcileKubernetesRestoreRecreateForcesSurvivingPool(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + // Deleting only the Kubernetes application leaves its KubernetesNodes + // pool's HelmRelease behind, while the pool's MachineDeployment went + // with the CAPI Cluster. + pool := unstr(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, k8sTestNS, "kubernetes-nodes-gone-md0", map[string]any{"spec": map[string]any{"interval": "5m"}}) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Create(ctx, pool, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + driveRecreateToEtcdJob(t, run, "uid-1") + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + ann := run.obj(helmReleaseGVR, "kubernetes-nodes-gone-md0").GetAnnotations() + if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/forceAt"] != ann["reconcile.fluxcd.io/requestedAt"] { + t.Fatalf("the surviving pool is not forced to recreate its MachineDeployment: %v", ann) + } +} + +func TestReconcileKubernetesBackupRefusesClusterStoppedByRestore(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj3"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesReplicasBeforeAnnotation + `":"2"}},"spec":{"replicas":0}}`) + if _, err := env.backup.Interface.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Patch(ctx, k8sTestRelease, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + for range 2 { + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a cluster stopped by an unfinished restore was backed up: %+v", got.Status) + } + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj3-backup"}, &batchv1.Job{}); err == nil { + t.Fatal("the etcd prefix of a stopped cluster was dumped") + } +} + +func TestStopKubernetesClusterWaitsForTerminatingControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + cp := unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"kamaji.clastix.io/name": k8sTestRelease}}}, + "status": map[string]any{"replicas": int64(0)}, + }) + if _, err := run.env.restore.Interface.Resource(deploymentGVR).Namespace(k8sTestNS).Create(ctx, cp, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "apiserver-0", Labels: map[string]string{"kamaji.clastix.io/name": k8sTestRelease}}} + if err := run.env.client.Create(ctx, pod); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster reported stopped while a control-plane Pod is still terminating") + } + if err := run.env.client.Delete(ctx, pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped once its control plane was gone") + } +} + +func TestKubernetesJobReachedWriteConservativeBranches(t *testing.T) { + for _, tc := range []struct { + name string + status corev1.ContainerStatus + want bool + }{ + {"running", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, true}, + {"restarted after writing", corev1.ContainerStatus{Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}, + LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: kubernetesJobWriting}}}, true}, + {"never ran", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, false}, + {"failed before writing", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1}}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "p", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{tc.status}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + got, err := run.env.restore.kubernetesJobReachedWrite(context.Background(), run.get(), "rj-restore") + if err != nil || got != tc.want { + t.Fatalf("reached write = %v (%v), want %v", got, err, tc.want) + } + }) + } +} + +// TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed covers a Cluster +// API that no longer serves the version the driver lists. Reading that as a +// cluster without workers would let the volumes path delete DataVolumes that +// running workers still mount. +func TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed(t *testing.T) { + for _, tc := range []struct{ resource, kind string }{ + {"machinedeployments", "MachineDeployment"}, + {"machines", "Machine"}, + } { + t.Run(tc.resource, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", tc.resource, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: tc.kind}, SearchedVersions: []string{"v1beta1"}} + }) + run.step() + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "Cluster API is not served") { + t.Fatalf("restore not failed on an unserved Cluster API: %+v", got.Status) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatalf("backed-up DataVolume deleted: %v", err) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } + }) + } +} + +// TestReconcileKubernetesRestoreRefusesRecreatedCluster covers an application +// deleted after the backup and created again under the same name. Its new CA +// and service-account key come back only through a rebuild, so an in-place +// restore must refuse before it changes anything. Backups without a recorded +// UID are told apart by the datastore schema. +func TestReconcileKubernetesRestoreRefusesRecreatedCluster(t *testing.T) { + for _, tc := range []struct { + name string + edit func(map[string]string) + }{ + {"recorded UID", func(md map[string]string) { md[kubernetesControlPlaneUIDMetaKey] = "uid-of-the-deleted-cluster" }}, + {"schema of a backup without the UID", func(md map[string]string) { + delete(md, kubernetesControlPlaneUIDMetaKey) + md[kubernetesDatastorePrefixMetaKey] = "uid-of-the-deleted-cluster" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + backup := k8sBackupForRestore() + tc.edit(backup.Spec.DriverMetadata) + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "deleted and created again") { + t.Fatalf("restore into a re-created cluster not refused: %+v", got.Status) + } + if kubernetesStateFromRestoreJob(got) != nil { + t.Fatal("restore mode recorded for a refused restore") + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended by a refused restore") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool scaled to %d by a refused restore", n) + } + }) + } +} + +// TestFailedKubernetesRestoreResumesWithoutClusterAPI covers Cluster API going +// unserved after the workers were stopped: the pools cannot be read back, but +// the autoscaler and the suspended HelmReleases must still be put back. +func TestFailedKubernetesRestoreResumesWithoutClusterAPI(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // workers stopping, the Machine still there + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not stopped: %d", n) + } + for _, res := range []string{"machinedeployments", "machines"} { + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", res, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: "MachineDeployment"}, SearchedVersions: []string{"v1beta1"}} + }) + } + run.step() + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore not failed: %+v", got.Status) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 1 { + t.Fatalf("autoscaler left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } +} + +// TestRenderKubernetesTemplateRefusesUnexpectedSchema pins the allowlist every +// strategy Job is rendered through: the schema becomes the prefix the restore +// deletes in the etcd the whole tenant shares. +func TestRenderKubernetesTemplateRefusesUnexpectedSchema(t *testing.T) { + tmpl := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "etcd", Args: []string{"{{ .Datastore.Prefix }} {{ .Datastore.SourcePrefix }}"}}}}} + render := func(prefix, source string) error { + _, err := renderKubernetesTemplate(tmpl, k8sTestApp, k8sTestNS, kubernetesModeRestore, "key", kubernetesDatastore{Prefix: prefix, SourcePrefix: source}, nil) + return err + } + for _, ok := range []string{"0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10", "tenant_root_kubernetes_bk1"} { + if err := render(ok, ok); err != nil { + t.Errorf("schema %q refused: %v", ok, err) + } + } + if err := render("tenant_a_kubernetes_c1", ""); err != nil { + t.Errorf("a Job without a source schema refused: %v", err) + } + for _, bad := range []string{"", "a/b", "/", "..", "a..b", "a.b", "Tenant_a", "a b", "a\nb", "a*"} { + if render(bad, "tenant_a_kubernetes_c1") == nil { + t.Errorf("schema %q accepted", bad) + } + if bad != "" && render("tenant_a_kubernetes_c1", bad) == nil { + t.Errorf("source schema %q accepted", bad) + } + } +} + +// TestReconcileKubernetesRestoreRefusesEditedPrefix covers a Backup whose +// recorded prefix was edited: the restore fails in its preflight, before any +// Job runs against the shared etcd or the cluster is stopped. +func TestReconcileKubernetesRestoreRefusesEditedPrefix(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] = "tenant_a_kubernetes_c1/.." + run := newK8sRestoreRun(t, "", backup) + run.step() // StartedAt + run.step() // preflight + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("restore from an edited prefix not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := run.env.client.List(context.Background(), jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("Jobs created for an edited prefix: %d %v", len(jobs.Items), err) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended") + } +} + +// TestReconcileKubernetesBackupRefusesUnexpectedSchema covers a live +// TenantControlPlane reporting a schema outside the allowlist. +func TestReconcileKubernetesBackupRefusesUnexpectedSchema(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + tcp, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(ctx, k8sTestRelease, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(tcp.Object, "tenant_a/kubernetes_c1", "status", "storage", "setup", "schema"); err != nil { + t.Fatal(err) + } + if _, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(ctx, tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + for i := 0; i < 4 && get().Status.Phase != backupsv1alpha1.BackupJobPhaseFailed; i++ { + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("backup of an unexpected schema not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := env.client.List(ctx, jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("dump Job created for an unexpected schema: %d %v", len(jobs.Items), err) + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_restore.go b/internal/backupcontroller/kubernetesstrategy_restore.go new file mode 100644 index 000000000..830a25927 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_restore.go @@ -0,0 +1,1129 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + // kubernetesCondMode records on the RestoreJob whether it rebuilds a + // deleted cluster or rolls a live one back (Reason), and which cluster + // (Message). It lives in status, which the creator of a RestoreJob cannot + // set, so a RestoreJob cannot claim a live cluster is gone. + kubernetesCondMode = "KubernetesRestoreMode" + kubernetesRestorePhaseLabel = "kubernetes.strategy.backups.cozystack.io/restore-phase" + + // What a restore changes on a live cluster is recorded on the objects it + // changes rather than on the RestoreJob, so the cluster can be put back + // after the RestoreJob is deleted, and a retry after a failed restore + // keeps the counts from before the first one rather than the zeros. + kubernetesReplicasBeforeAnnotation = "kubernetes.strategy.backups.cozystack.io/replicas-before-restore" + kubernetesSuspendedByAnnotation = "kubernetes.strategy.backups.cozystack.io/suspended-by-restore" + kubernetesReplicasUnset = "unset" + + kubernetesRestoreModeRecreate = "recreate" + kubernetesRestoreModeInPlace = "in-place" + + kubernetesPhaseSecrets = "secrets" + kubernetesPhaseVolumes = "volumes" + + machineExcludeDrainAnnotation = "machine.cluster.x-k8s.io/exclude-node-draining" + machineExcludeVolumeDetachAnnotation = "machine.cluster.x-k8s.io/exclude-wait-for-node-volume-detach" + + kubernetesCondVerified = "KubernetesBackupVerified" + kubernetesCondRecreated = "KubernetesClusterRecreated" + kubernetesCondStopped = "KubernetesClusterStopped" + kubernetesCondSecretsRestored = "KubernetesSecretsRestored" + kubernetesCondEtcdRestored = "KubernetesEtcdRestored" + kubernetesCondVolumesRestored = "KubernetesVolumesRestored" + // kubernetesCondEtcdWriteFailed marks a restore whose etcd Job failed + // after it started replacing the prefix. The control plane is then left + // stopped, by a failure and by the RestoreJob's deletion alike. + kubernetesCondEtcdWriteFailed = "KubernetesEtcdWriteFailed" + // kubernetesCondEtcdJobStarted records that a container of the restore + // Job was seen running, which is the only trace left once the Job's + // deadline has deleted its Pods. + kubernetesCondEtcdJobStarted = "KubernetesEtcdJobStarted" + + // The restore Job writes this to its termination message just before it + // deletes the prefix, so a failed Job tells whether the prefix is still + // intact. + kubernetesJobWriting = "writing" +) + +// KubernetesRestoreOptions is the typed shape of RestoreJob.Spec.Options for +// the Kubernetes driver. Both halves of the backup are restored unless one is +// switched off. +type KubernetesRestoreOptions struct { + // Etcd restores the cluster's etcd prefix (its API objects). + Etcd *bool `json:"etcd,omitempty"` + // Volumes restores the DataVolumes backing the cluster's PersistentVolumes. + Volumes *bool `json:"volumes,omitempty"` +} + +func (o KubernetesRestoreOptions) restoreEtcd() bool { return boolDefault(o.Etcd, true) } +func (o KubernetesRestoreOptions) restoreVolumes() bool { return boolDefault(o.Volumes, true) } + +func parseKubernetesRestoreOptions(opts *runtime.RawExtension) (KubernetesRestoreOptions, error) { + var out KubernetesRestoreOptions + if opts == nil || len(opts.Raw) == 0 { + return out, nil + } + if err := json.Unmarshal(opts.Raw, &out); err != nil { + return KubernetesRestoreOptions{}, fmt.Errorf("decode restoreJob.spec.options: %w", err) + } + if !out.restoreEtcd() && !out.restoreVolumes() { + return KubernetesRestoreOptions{}, fmt.Errorf("restoreJob.spec.options disables both etcd and volumes; nothing to restore") + } + return out, nil +} + +type kubernetesRestoreState struct { + Mode string `json:"mode"` + Release string `json:"release"` +} + +func kubernetesAutoscalerName(release string) string { + return release + "-cluster-autoscaler" +} + +func kubernetesBackupDataVolumes(b *backupsv1alpha1.Backup) []string { + raw := b.Spec.DriverMetadata[kubernetesDataVolumesMetaKey] + if raw == "" { + return nil + } + return strings.Split(raw, ",") +} + +func kubernetesBackupHelmReleases(b *backupsv1alpha1.Backup) ([]map[string]any, error) { + var out []map[string]any + if err := json.Unmarshal([]byte(b.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey]), &out); err != nil { + return nil, fmt.Errorf("decode recorded HelmReleases: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("the Backup records no HelmReleases") + } + return out, nil +} + +// kubernetesPreRestoreKey is where the restore Job keeps the prefix it is +// about to replace. The Job writes it only when none is there and deletes it +// once the restore succeeds, so after a failed restore it holds the prefix +// as it was before the first failed attempt, and a retry does not overwrite +// it with a partial one. It is fixed per backup, so the Backup's cleanup +// removes it. +func kubernetesPreRestoreKey(objectKey string) string { + return objectKey + ".pre-restore" +} + +// reconcileKubernetesRestore restores a tenant Kubernetes application in +// place, under its own name and namespace. +// +// When the application is gone, the restore rebuilds it: control-plane PKI +// Secrets first (the etcd Job authenticates with one of them), then the etcd +// prefix, the volumes, and finally the HelmReleases, so the control plane +// comes up on restored state and the existing CA. +// +// When the application is live, the restore stops what the selected halves +// need stopped — the workers for volumes, so the volumes are detached, then +// the control plane for etcd, so nothing writes while the prefix is replaced +// — writes the state back, and returns the cluster to the replica counts it +// had. +// +// Nothing is changed before the backup's artifacts are confirmed to exist. +func (r *RestoreJobReconciler) reconcileKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || + rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(backup.Spec.ApplicationRef); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + appName := backup.Spec.ApplicationRef.Name + if t := rj.Spec.TargetApplicationRef; t != nil && ((t.Name != "" && t.Name != appName) || (t.Kind != "" && t.Kind != kubernetesAppKind)) { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the application the backup was taken from") + } + if rj.Namespace != backup.Namespace { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the namespace the backup was taken from") + } + opts, err := parseKubernetesRestoreOptions(rj.Spec.Options) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if rj.Status.StartedAt == nil { + base := rj.DeepCopy() + now := metav1.Now() + rj.Status.StartedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning + if err := r.Status().Patch(ctx, rj, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueRestoreStrategyNotReady(ctx, rj, backup.Spec.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVerified) { + done, msg, err := r.verifyKubernetesBackup(ctx, rj, backup, strategy, opts) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVerified, "the backup's artifacts are present") + } + + state := kubernetesStateFromRestoreJob(rj) + if state == nil { + return r.captureKubernetesRestoreState(ctx, rj, backup) + } + + if state.Mode == kubernetesRestoreModeInPlace { + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, opts, state) + } + return r.reconcileKubernetesRecreate(ctx, rj, backup, strategy, opts, state) +} + +func kubernetesStateFromRestoreJob(rj *backupsv1alpha1.RestoreJob) *kubernetesRestoreState { + c := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondMode) + if c == nil { + return nil + } + mode := kubernetesRestoreModeInPlace + if c.Reason == "Recreate" { + mode = kubernetesRestoreModeRecreate + } + return &kubernetesRestoreState{Mode: mode, Release: c.Message} +} + +// verifyKubernetesBackup confirms, before anything is stopped or deleted, +// that the Velero backup completed and still exists, and that the etcd dump +// object is in the bucket. The dump is checked by a verify-mode run of the +// strategy Job, the only component holding the bucket credentials. +func (r *RestoreJobReconciler) verifyKubernetesBackup(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions) (bool, string, error) { + name := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if name == "" { + return false, "the Backup records no Velero backup", nil + } + vb := &velerov1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb); err != nil { + if apierrors.IsNotFound(err) { + return false, fmt.Sprintf("Velero backup %s/%s no longer exists", veleroNamespace, name), nil + } + return false, "", err + } + if vb.Status.Phase != velerov1.BackupPhaseCompleted { + return false, fmt.Sprintf("Velero backup %s/%s is %q, not Completed", veleroNamespace, name, vb.Status.Phase), nil + } + if !opts.restoreEtcd() { + return true, "", nil + } + return r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeVerify, rj.Name+"-verify", kubernetesBackupDatastore(backup)) +} + +// captureKubernetesRestoreState decides between rebuilding a deleted cluster +// and restoring a live one. +func (r *RestoreJobReconciler) captureKubernetesRestoreState(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + release := kubernetesReleaseName(backup.Spec.ApplicationRef.Name) + reason := "Recreate" + _, err := r.Resource(helmReleaseGVR).Namespace(rj.Namespace).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return ctrl.Result{}, err + default: + reason = "InPlace" + mismatch, err := r.kubernetesClusterMismatch(ctx, rj.Namespace, release, backup) + if err != nil { + return ctrl.Result{}, err + } + if mismatch != "" { + return r.markRestoreJobFailed(ctx, rj, mismatch) + } + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondMode, + Status: metav1.ConditionTrue, + Reason: reason, + Message: release, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +// capiNotServed reports a list of a Cluster API resource that failed because +// the apiserver does not serve that resource at the version the driver asks +// for: a NotFound on a collection, or no RESTMapper match. +func capiNotServed(err error) bool { + return err != nil && (apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err)) +} + +// kubernetesClusterMismatch reports why the live cluster is not the one the +// backup was taken from, or "" when it is or cannot be told apart. A cluster +// deleted and created again under the same name has a new CA, service-account +// key and Talos secrets, which only a rebuild restores: writing the old prefix +// into it would leave every token and node signed by keys it no longer has. +// Backups that predate the recorded UID fall back to the datastore schema, +// which tells the two apart only on a Kamaji that keys it on the UID. +func (r *RestoreJobReconciler) kubernetesClusterMismatch(ctx context.Context, ns, release string, backup *backupsv1alpha1.Backup) (string, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return "", nil + } + if err != nil { + return "", err + } + md := backup.Spec.DriverMetadata + recorded, live := md[kubernetesControlPlaneUIDMetaKey], string(tcp.GetUID()) + if recorded == "" { + recorded = md[kubernetesDatastorePrefixMetaKey] + live, _, _ = unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + } + if recorded == "" || live == "" || recorded == live { + return "", nil + } + return fmt.Sprintf("TenantControlPlane %s/%s is not the cluster this backup was taken from (%s, backup has %s): it was deleted and created again under the same name, so an in-place restore would write the old state over a new CA and service-account key; delete the application and restore again to rebuild it with the backed-up PKI", ns, release, live, recorded), nil +} + +func (r *RestoreJobReconciler) listClusterMachineDeployments(ctx context.Context, ns, release string) ([]unstructured.Unstructured, error) { + list, err := r.Resource(machineDeploymentGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + var out []unstructured.Unstructured + for _, md := range list.Items { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster == release { + out = append(out, md) + } + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Rebuild a deleted cluster +// --------------------------------------------------------------------------- + +// reconcileKubernetesRecreate rebuilds a deleted cluster. The control-plane +// PKI and the volumes come back first, then the cluster's own HelmRelease. +// Kamaji gives the new TenantControlPlane a fresh datastore schema (keyed on +// its UID), so the etcd half can only be written once that schema exists: +// the restore then stops the new control plane like a live one, writes the +// dump into the new prefix, and starts it again. The worker pools are +// recreated last, so their workers join the restored cluster rather than an +// empty one. +func (r *RestoreJobReconciler) reconcileKubernetesRecreate(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondSecretsRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseSecrets, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondSecretsRestored, "control-plane Secrets restored") + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + } + + if !opts.restoreEtcd() { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases, true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup without its etcd state") + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondRecreated) { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[:1], false); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + ready, err := r.tenantControlPlaneSchemaReady(ctx, rj.Namespace, state.Release) + if err != nil { + return ctrl.Result{}, err + } + if ready { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondRecreated, "the control plane is up with a datastore schema") + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondRecreated, + Status: metav1.ConditionFalse, + Reason: "Waiting", + Message: "waiting for TenantControlPlane " + state.Release + " to report its datastore schema", + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondRecreated) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf("TenantControlPlane %s/%s reported no datastore schema within %s", rj.Namespace, state.Release, restorePreparationDeadline)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, KubernetesRestoreOptions{Etcd: new(true), Volumes: new(false)}, state) +} + +func (r *RestoreJobReconciler) tenantControlPlaneSchemaReady(ctx context.Context, ns, release string) (bool, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return false, client.IgnoreNotFound(err) + } + schema, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + return schema != "", nil +} + +// createKubernetesHelmReleases recreates the recorded HelmReleases. A worker +// pool's HelmRelease outlives the deletion of its cluster (KubernetesNodes is +// an application of its own), but its MachineDeployment was garbage-collected +// with the CAPI Cluster, and nothing re-applies an unchanged release. One +// that exists is therefore forced through a Helm upgrade, which recreates +// the objects its release is missing. The cluster's own release is created +// while the rebuild polls for its schema, so it passes force=false: forcing +// it on every poll would re-run its chart while Kamaji brings it up. +func (r *RestoreJobReconciler) createKubernetesHelmReleases(ctx context.Context, ns string, releases []map[string]any, force bool) error { + for _, hr := range releases { + obj := &unstructured.Unstructured{Object: hr} + obj.SetNamespace(ns) + _, err := r.Resource(helmReleaseGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}) + if err == nil { + continue + } + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("recreate HelmRelease %s/%s: %w", ns, obj.GetName(), err) + } + if !force { + continue + } + now := time.Now().UTC().Format(time.RFC3339Nano) + patch, merr := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "reconcile.fluxcd.io/requestedAt": now, + "reconcile.fluxcd.io/forceAt": now, + }}}) + if merr != nil { + return merr + } + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, obj.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("force HelmRelease %s/%s to reconcile: %w", ns, obj.GetName(), err) + } + } + return nil +} + +// --------------------------------------------------------------------------- +// Restore a live cluster +// --------------------------------------------------------------------------- + +func (r *RestoreJobReconciler) reconcileKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondStopped) { + stopped, waiting, err := r.stopKubernetesCluster(ctx, rj, backup, opts, state.Release) + if capiNotServed(err) { + // Reading this as "no workers" would delete volumes that running + // workers still mount. Workers are listed before anything is + // stopped, so the resume below has only what it already stopped + // to put back. + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("Cluster API is not served at %s: %v", machineDeploymentGVR.GroupVersion(), err)) + } + if err != nil { + // Retried like any other wait, under the same deadline: a single + // apiserver blip must not abort a restore. + stopped, waiting = false, fmt.Sprintf("a retry after an API error: %v", err) + } + if stopped { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondStopped, "cluster stopped for restore") + } + // The condition's transition time marks when stopping began: it is + // kept while the status stays False, whatever the message says. + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondStopped, + Status: metav1.ConditionFalse, + Reason: "Stopping", + Message: "waiting for " + waiting, + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("the cluster did not stop within %s: still waiting for %s", restorePreparationDeadline, waiting)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if res, next, err := r.kubernetesRestoreData(ctx, rj, backup, strategy, opts, state); !next { + return res, err + } + + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + return ctrl.Result{}, err + } + if state.Mode == kubernetesRestoreModeRecreate { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[1:], true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup") + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster restored in place") +} + +// stopKubernetesCluster suspends the cluster's HelmReleases so Flux does not +// undo the scale-down, then stops the workers when volumes are restored — +// deleting the data DataVolumes and PVCs the backup holds once nothing mounts +// them — and the control plane when etcd is. Workers go first: CAPI removes a +// Machine's Node through the tenant API. It reports whether all of that has +// settled and, if not, what it is waiting for. +func (r *RestoreJobReconciler) stopKubernetesCluster(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts KubernetesRestoreOptions, release string) (bool, string, error) { + ns := rj.Namespace + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil { + return false, "", err + } + for _, hr := range kubernetesHelmReleaseNames(mds, release) { + if err := r.suspendHelmReleaseForRestore(ctx, ns, hr); err != nil { + return false, "", err + } + } + + if opts.restoreVolumes() { + if err := r.stopReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + return false, "", err + } + for _, md := range mds { + if err := r.stopReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + return false, "", err + } + } + remaining, err := r.releaseClusterMachines(ctx, ns, release) + if err != nil { + return false, "", err + } + if remaining > 0 { + return false, fmt.Sprintf("%d worker Machine(s) to be deleted", remaining), nil + } + var pending []string + for _, name := range kubernetesBackupDataVolumes(backup) { + dvGone, err := r.deleteDataVolume(ctx, ns, name) + if err != nil { + return false, "", err + } + pvcGone, err := r.deletePVC(ctx, ns, name) + if err != nil { + return false, "", err + } + if !dvGone || !pvcGone { + pending = append(pending, name) + } + } + if len(pending) > 0 { + return false, "data volumes to be deleted: " + strings.Join(pending, ", "), nil + } + } + + if opts.restoreEtcd() { + if err := r.stopReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + return false, "", err + } + cp, err := r.Resource(deploymentGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return false, "", err + default: + if n, _, _ := unstructured.NestedInt64(cp.Object, "status", "replicas"); n > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to stop", n), nil + } + // status.replicas leaves out Pods that are still terminating, and + // a terminating kube-apiserver can still write to etcd. + selector, _, _ := unstructured.NestedStringMap(cp.Object, "spec", "selector", "matchLabels") + if len(selector) > 0 { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels(selector)); err != nil { + return false, "", err + } + if len(pods.Items) > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to terminate", len(pods.Items)), nil + } + } + } + } + return true, "", nil +} + +// releaseClusterMachines lets the cluster's Machines finish deleting without +// draining their Nodes or waiting for volumes to detach: the workers are +// recreated after the restore, and a drain can block on a PodDisruptionBudget. +// It returns how many Machines remain. +func (r *RestoreJobReconciler) releaseClusterMachines(ctx context.Context, ns, release string) (int, error) { + machines, err := r.Resource(machineGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: clusterNameLabel + "=" + release}) + if err != nil { + return 0, err + } + patch := []byte(`{"metadata":{"annotations":{"` + machineExcludeDrainAnnotation + `":"","` + machineExcludeVolumeDetachAnnotation + `":""}}}`) + for _, m := range machines.Items { + ann := m.GetAnnotations() + if _, ok := ann[machineExcludeDrainAnnotation]; ok { + if _, ok := ann[machineExcludeVolumeDetachAnnotation]; ok { + continue + } + } + if _, err := r.Resource(machineGVR).Namespace(ns).Patch(ctx, m.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + return 0, err + } + } + return len(machines.Items), nil +} + +// suspendHelmReleaseForRestore suspends a HelmRelease and marks it as +// suspended by the restore. One already suspended is left alone and unmarked, +// so resuming the cluster does not resume it. +func (r *RestoreJobReconciler) suspendHelmReleaseForRestore(ctx context.Context, ns, name string) error { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + if suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend"); suspended { + return nil + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":"true"}},"spec":{"suspend":true}}`) + _, err = r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// stopReplicas scales an object to zero, first recording its replica count +// on it. A count already recorded is kept: it was taken before an earlier +// restore stopped the object. +func (r *RestoreJobReconciler) stopReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + annotations := map[string]any{} + if _, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; !recorded { + before := kubernetesReplicasUnset + if n, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas"); found { + before = strconv.FormatInt(n, 10) + } + annotations[kubernetesReplicasBeforeAnnotation] = before + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": annotations}, + "spec": map[string]any{"replicas": 0}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeReplicas puts back the replica count stopReplicas recorded — removing +// the field when it was unset, so its owner (the autoscaler, for a +// MachineDeployment) takes it back — and drops the record. +func (r *RestoreJobReconciler) resumeReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + before, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation] + if !recorded { + return nil + } + var replicas any + if before != kubernetesReplicasUnset { + n, err := strconv.ParseInt(before, 10, 64) + if err != nil { + return fmt.Errorf("%s %s/%s: bad %s %q", gvr.Resource, ns, name, kubernetesReplicasBeforeAnnotation, before) + } + replicas = n + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{kubernetesReplicasBeforeAnnotation: nil}}, + "spec": map[string]any{"replicas": replicas}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeKubernetesCluster undoes stopKubernetesCluster from the records on +// the objects themselves, so it needs nothing from the RestoreJob and is safe +// to call more than once. +func (r *RestoreJobReconciler) resumeKubernetesCluster(ctx context.Context, ns, release string) error { + var errs []string + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil && !capiNotServed(err) { + return err + } + if err := r.resumeReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + errs = append(errs, err.Error()) + } + for _, md := range mds { + if err := r.resumeReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + errs = append(errs, err.Error()) + } + } + if err := r.resumeReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + errs = append(errs, err.Error()) + } + for _, name := range kubernetesHelmReleaseNames(mds, release) { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + continue + } + if _, marked := hr.GetAnnotations()[kubernetesSuspendedByAnnotation]; !marked { + continue + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":null}},"spec":{"suspend":false}}`) + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("resume cluster: %s", strings.Join(errs, "; ")) + } + return nil +} + +// failKubernetesInPlace fails a live-cluster restore. The cluster is resumed +// unless the etcd prefix may have been partly replaced: bringing a control +// plane up on that would serve a mix of old and restored objects, so it is +// left stopped for a retry or a manual repair. +func (r *RestoreJobReconciler) failKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, msg string) (ctrl.Result, error) { + if apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) { + return r.markRestoreJobFailed(ctx, rj, msg) + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + msg = fmt.Sprintf("%s; the cluster could not be fully resumed: %v", msg, err) + } + return r.markRestoreJobFailed(ctx, rj, msg) +} + +// cleanupKubernetesRestore runs when a RestoreJob is deleted. A live cluster +// it stopped is resumed, unless its etcd prefix may be partly replaced, which +// is left for the next restore to repair and reported as a Warning event. +func (r *RestoreJobReconciler) cleanupKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob) { + state := kubernetesStateFromRestoreJob(rj) + if state == nil || state.Release == "" || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded { + return + } + partial := apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) + // A restore Job still running may be rewriting the prefix right now; + // one that failed after it started writing left it partial. Either way + // the control plane must not come up on it. The Job is deleted so it + // stops writing; the garbage collector would only get to it later. + job := &batchv1.Job{} + switch err := r.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: jobNameForRestoreJob(rj)}, job); { + case err == nil && jobConditionState(job) != batchv1.JobComplete: + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, job.Name) + if werr != nil || wrote || jobConditionState(job) != batchv1.JobFailed { + partial = true + } + policy := metav1.DeletePropagationBackground + _ = r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &policy}) + case err != nil && !apierrors.IsNotFound(err): + partial = true + } + if partial { + if r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "ClusterLeftStopped", + "%s/%s stays stopped: its etcd prefix may be partly restored; run a new restore from a Backup", rj.Namespace, state.Release) + } + return + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil && r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "CleanupFailed", "resume %s/%s: %v", rj.Namespace, state.Release, err) + } +} + +// --------------------------------------------------------------------------- +// Steps shared by both modes +// --------------------------------------------------------------------------- + +// kubernetesRestoreData writes back the selected halves, etcd before volumes. +// next is true when both are done and the caller may move on; otherwise the +// returned result is the reconcile's. +func (r *RestoreJobReconciler) kubernetesRestoreData(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, bool, error) { + if opts.restoreEtcd() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdRestored) { + ds, err := resolveKubernetesDatastore(ctx, r.Interface, rj.Namespace, backup.Spec.ApplicationRef.Name) + if err != nil { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, fmt.Sprintf("resolve the cluster's datastore: %v", err), false) + return res, false, rerr + } + ds.SourcePrefix = backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] + done, msg, err := r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeRestore, jobNameForRestoreJob(rj), *ds) + if msg != "" && err == nil { + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, jobNameForRestoreJob(rj)) + if werr != nil { + return ctrl.Result{}, false, werr + } + if !wrote { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, "etcd restore failed before it changed the prefix: "+msg, false) + return res, false, rerr + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondEtcdWriteFailed, + Status: metav1.ConditionTrue, + Reason: "RestoreJobFailed", + Message: msg, + }) + msg = fmt.Sprintf("etcd restore failed: %s. The prefix may be partly restored, so the control plane is left stopped. "+ + "A copy of the prefix from before the first restore of this backup that did not finish is kept at %s in the backup bucket; run a new RestoreJob to retry", + msg, kubernetesPreRestoreKey(backup.Spec.DriverMetadata[kubernetesObjectMetaKey])) + } + if err == nil && msg == "" && !done && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted) { + started, serr := r.kubernetesJobStarted(ctx, rj.Namespace, jobNameForRestoreJob(rj)) + if serr != nil { + return ctrl.Result{}, false, serr + } + if started { + res, cerr := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdJobStarted, "the etcd restore Job is running") + return res, false, cerr + } + } + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdRestored, "etcd prefix restored") + return res, false, err + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, kubernetesReleaseName(backup.Spec.ApplicationRef.Name)) + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + return res, false, err + } + return ctrl.Result{}, true, nil +} + +// kubernetesStepResult turns a step's outcome into the reconcile result: an +// API error is retried, a failure message fails the restore (resuming a live +// cluster first where that is safe), and an unfinished step is polled. +func (r *RestoreJobReconciler) kubernetesStepResult(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, err error, failure string, done bool) (ctrl.Result, error) { + switch { + case err != nil: + return ctrl.Result{}, err + case failure != "": + // A rebuild stops its new control plane for the etcd half the same + // way a live restore does, so both put back whatever they stopped. + if state != nil { + return r.failKubernetesInPlace(ctx, rj, state, failure) + } + return r.markRestoreJobFailed(ctx, rj, failure) + case !done: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + return ctrl.Result{}, nil +} + +// kubernetesJobReachedWrite reports whether any Pod of the restore Job may +// have changed the prefix: one running now, or one that terminated with the +// marker the script writes just before it deletes the prefix. A container +// that never ran (an image it could not pull) wrote nothing. When the Job's +// Pods are gone (its deadline deletes them), the answer is whether any of +// them was ever seen running. +func (r *RestoreJobReconciler) kubernetesJobReachedWrite(ctx context.Context, rj *backupsv1alpha1.RestoreJob, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(rj.Namespace), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + if len(pods.Items) == 0 { + return apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted), nil + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + switch { + case cs.State.Running != nil: + return true, nil + case cs.State.Terminated != nil: + if strings.TrimSpace(cs.State.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + case cs.LastTerminationState.Terminated != nil: + if strings.TrimSpace(cs.LastTerminationState.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + } + } + } + return false, nil +} + +// kubernetesJobStarted reports whether a container of the restore Job has +// run, which is what kubernetesJobReachedWrite falls back on once the Job's +// Pods are gone. +func (r *RestoreJobReconciler) kubernetesJobStarted(ctx context.Context, ns, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Running != nil || cs.State.Terminated != nil || cs.LastTerminationState.Terminated != nil { + return true, nil + } + } + } + return false, nil +} + +// runKubernetesJob runs the strategy Job in the given mode under the given +// name against ds. It returns done once the Job completed, or a failure +// message once it failed. +func (r *RestoreJobReconciler) runKubernetesJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, mode, name string, ds kubernetesDatastore) (bool, string, error) { + if ds.Prefix == "" || len(ds.Endpoints) == 0 { + return false, "the Backup records no etcd datastore prefix or endpoints", nil + } + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, rj.Namespace, + mode, backup.Spec.DriverMetadata[kubernetesObjectMetaKey], ds, kubernetesStrategyParameters(backup)) + if err != nil { + return false, fmt.Sprintf("failed to template Kubernetes strategy: %v", err), nil + } + job, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, rj, rj.Namespace, name, + map[string]string{ + kubernetesLabelMode: mode, + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + }, rendered) + if err != nil { + return false, "", err + } + switch jobConditionState(job) { + case batchv1.JobComplete: + return true, "", nil + case batchv1.JobFailed: + msg := jobFailureMessage(job) + if msg == "" { + msg = fmt.Sprintf("%s Job reported Failed", mode) + } + return false, msg, nil + } + return false, "", nil +} + +// runKubernetesVeleroRestore restores one phase of the Velero backup: +// control-plane Secrets, or the cluster's data DataVolumes and PVCs. It +// returns done once the Velero Restore completed. +func (r *RestoreJobReconciler) runKubernetesVeleroRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, phase, release string) (bool, string, error) { + veleroBackup := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if veleroBackup == "" { + return false, "the Backup records no Velero backup", nil + } + labels := map[string]string{ + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + kubernetesRestorePhaseLabel: phase, + } + restores := &velerov1.RestoreList{} + if err := r.List(ctx, restores, client.InNamespace(veleroNamespace), client.MatchingLabels(labels)); err != nil { + return false, "", err + } + if len(restores.Items) == 0 { + spec, err := kubernetesVeleroRestoreSpec(strategy, backup, phase, release) + if err != nil { + return false, fmt.Sprintf("failed to build Velero Restore: %v", err), nil + } + spec.BackupName = veleroBackup + cm, err := r.kubernetesResourceModifiers(ctx, rj, phase, labels) + if err != nil { + return false, "", err + } + spec.ResourceModifier = &corev1.TypedLocalObjectReference{APIGroup: new(""), Kind: "ConfigMap", Name: cm.Name} + restore := &velerov1.Restore{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-%s-", rj.Namespace, rj.Name, phase), + Namespace: veleroNamespace, + Labels: labels, + }, + Spec: *spec, + } + if err := r.Create(ctx, restore); err != nil { + return false, "", err + } + if r.Recorder != nil { + r.Recorder.Event(rj, corev1.EventTypeNormal, "VeleroRestoreCreated", fmt.Sprintf("Created Velero Restore %s/%s", veleroNamespace, restore.Name)) + } + return false, "", nil + } + restore := &restores.Items[0] + switch restore.Status.Phase { + case velerov1.RestorePhaseCompleted: + return true, "", nil + case velerov1.RestorePhaseFailed, velerov1.RestorePhasePartiallyFailed, velerov1.RestorePhaseFailedValidation: + msg := fmt.Sprintf("Velero Restore %s/%s finished %s", restore.Namespace, restore.Name, restore.Status.Phase) + if restore.Status.FailureReason != "" { + msg += ": " + restore.Status.FailureReason + } + return false, msg, nil + } + return false, "", nil +} + +// kubernetesVeleroRestoreSpec starts from the strategy's restore spec and +// narrows it to one phase. The selectors mirror the ones the shipped backup +// spec uses, so a phase restores exactly its share of the backup. +func kubernetesVeleroRestoreSpec(strategy *strategyv1alpha1.Kubernetes, backup *backupsv1alpha1.Backup, phase, release string) (*velerov1.RestoreSpec, error) { + spec := &velerov1.RestoreSpec{} + if strategy.Spec.Velero.RestoreSpec != nil { + rendered, err := template.Template(strategy.Spec.Velero.RestoreSpec, map[string]any{ + "Application": map[string]any{ + "metadata": map[string]any{"name": backup.Spec.ApplicationRef.Name, "namespace": backup.Namespace}, + "kind": backup.Spec.ApplicationRef.Kind, + }, + "Parameters": kubernetesStrategyParameters(backup), + }) + if err != nil { + return nil, err + } + spec = rendered + } + spec.IncludedNamespaces = []string{backup.Namespace} + spec.LabelSelector = nil + switch phase { + case kubernetesPhaseSecrets: + spec.IncludedResources = []string{"secrets"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{kamajiNameLabel: release}}, + {MatchLabels: map[string]string{helmInstanceLabel: release}}, + } + case kubernetesPhaseVolumes: + spec.IncludedResources = []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{clusterNameLabel: release}}, + } + spec.RestorePVs = new(true) + default: + return nil, fmt.Errorf("unknown restore phase %q", phase) + } + return spec, nil +} + +// kubernetesResourceModifiers creates the ConfigMap of resource modifiers for +// a phase. Restored Secrets drop their ownerReferences, which point at +// objects of the deleted cluster and would let the garbage collector delete +// them again; restored PVCs and DataVolumes let CDI adopt the restored claim. +func (r *RestoreJobReconciler) kubernetesResourceModifiers(ctx context.Context, rj *backupsv1alpha1.RestoreJob, phase string, labels map[string]string) (*corev1.ConfigMap, error) { + var rules []resourceModifierRule + switch phase { + case kubernetesPhaseSecrets: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"ownerReferences": nil}}) + if err != nil { + return nil, err + } + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: "secrets", ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + case kubernetesPhaseVolumes: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"annotations": map[string]string{cdiAllowClaimAdoption: "true"}}}) + if err != nil { + return nil, err + } + for _, gr := range []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} { + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: gr, ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + } + } + data, err := yaml.Marshal(resourceModifiers{Version: "v1", ResourceModifierRules: rules}) + if err != nil { + return nil, err + } + name := fmt.Sprintf("restore-modifiers-%s-%s-%s", rj.Namespace, rj.Name, phase) + if len(name) > 253 { + name = name[:253] + } + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: veleroNamespace, Labels: labels}, + Data: map[string]string{"resource-modifier-rules.yaml": string(data)}, + } + if err := r.Create(ctx, cm); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Namespace: veleroNamespace, Name: name}, existing); err != nil { + return nil, err + } + existing.Data = cm.Data + if err := r.Update(ctx, existing); err != nil { + return nil, err + } + return existing, nil + } + return cm, nil +} + +func (r *RestoreJobReconciler) setKubernetesRestoreCondition(ctx context.Context, rj *backupsv1alpha1.RestoreJob, condType, message string) (ctrl.Result, error) { + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: condType, + Status: metav1.ConditionTrue, + Reason: "Done", + Message: message, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *RestoreJobReconciler) markKubernetesRestoreSucceeded(ctx context.Context, rj *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) { + now := metav1.Now() + rj.Status.CompletedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "RestoreCompleted", + Message: message, + }) + return ctrl.Result{}, r.Status().Update(ctx, rj) +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 4d88ab8a1..9fd6780db 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -204,7 +205,10 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { - case cond != nil && cond.Status == metav1.ConditionTrue: + case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed: + return r.markBackupJobFailed(ctx, j, mariadbJobFailedMessage(ctx, r.Client, "Backup", mdbBackup.Namespace, mdbBackup.Name, cond)) + + case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete: // Backup completed successfully. Materialise the Cozystack artifact // and finalise the BackupJob. Idempotent: if the artifact already // exists (a previous reconcile created it and then raced on the @@ -231,34 +235,54 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 } return ctrl.Result{}, nil - case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed: - // Terminal failure. The operator marks Complete=False/JobFailed only - // after the underlying Job exhausts its backoff budget, so we can - // fail the BackupJob immediately rather than waiting for the - // driver-side deadline. - message := cond.Message - if message == "" { - message = "k8s.mariadb.com Backup terminal failure" - } - return r.markBackupJobFailed(ctx, j, message) - default: // Still running (Complete=False/JobRunning, Complete=False/JobNotComplete, - // JobSuspended, or no Complete condition yet). Apply a wall-clock - // deadline so a permanently-stuck Backup eventually fails the - // BackupJob instead of pinning it Running forever. + // JobSuspended, a reason this driver does not know, or no Complete + // condition yet). Apply a wall-clock deadline so a permanently-stuck + // Backup eventually fails the BackupJob instead of pinning it Running + // forever. if mariadbBackupDeadlineExceeded(j.Status.StartedAt) { - detail := "no Complete condition observed" - if cond != nil { - detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) - } return r.markBackupJobFailed(ctx, j, fmt.Sprintf( - "k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, detail)) + "k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, mariadbCompleteDetail(cond))) } return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil } } +// mariadbJobFailedMessage names the operator object whose Job failed and +// carries that Job's own Failed condition, because the operator's condition +// message is only "Failed". The operator names the Job after the object. +func mariadbJobFailedMessage(ctx context.Context, c client.Reader, kind, namespace, name string, cond *metav1.Condition) string { + message := fmt.Sprintf("k8s.mariadb.com/%s %s/%s failed (Complete=%s/%s)", kind, namespace, name, cond.Status, cond.Reason) + job := &batchv1.Job{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, job); err != nil { + if apierrors.IsNotFound(err) { + return fmt.Sprintf("%s; Job %s/%s not found", message, namespace, name) + } + return fmt.Sprintf("%s; reading Job %s/%s: %v", message, namespace, name, err) + } + for _, jc := range job.Status.Conditions { + if jc.Type == batchv1.JobFailed && jc.Status == corev1.ConditionTrue { + return fmt.Sprintf("%s; Job %s/%s: %s: %s", message, namespace, name, jc.Reason, jc.Message) + } + } + return fmt.Sprintf("%s; Job %s/%s reports no Failed condition", message, namespace, name) +} + +// mariadbCompleteDetail describes the last Complete condition for a deadline +// failure. The message matters for reason Failed, which the operator sets +// with its own cause ("Error creating Job", "Backup not complete") rather +// than from a Job. +func mariadbCompleteDetail(cond *metav1.Condition) string { + if cond == nil { + return "no Complete condition observed" + } + if cond.Message == "" { + return fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) + } + return fmt.Sprintf("Complete=%s/%s: %s", cond.Status, cond.Reason, cond.Message) +} + // mariadbBackupDeadlineExceeded reports whether enough wall-clock time has // elapsed since the BackupJob started that we should give up on a stuck // operator Backup. Returns false when StartedAt is nil so the very first @@ -577,7 +601,10 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { - case cond != nil && cond.Status == metav1.ConditionTrue: + case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed: + return r.markRestoreJobFailed(ctx, restoreJob, mariadbJobFailedMessage(ctx, r.Client, "Restore", mdbRestore.Namespace, mdbRestore.Name, cond)) + + case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete: now := metav1.Now() restoreJob.Status.CompletedAt = &now restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded @@ -592,23 +619,12 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } return ctrl.Result{}, nil - case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed: - message := cond.Message - if message == "" { - message = fmt.Sprintf("k8s.mariadb.com/Restore %s/%s terminal failure", mdbRestore.Namespace, mdbRestore.Name) - } - return r.markRestoreJobFailed(ctx, restoreJob, message) - default: deadline := options.effectiveRestoreDeadline() if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline { - detail := "no Complete condition observed" - if cond != nil { - detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) - } return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf( "k8s.mariadb.com/Restore did not complete within %s (%s; override via spec.options.restoreTimeoutSeconds)", - deadline, detail)) + deadline, mariadbCompleteDetail(cond))) } return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil } diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index 0e4f63b5e..f03b67d94 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -4,12 +4,15 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "strings" "testing" "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -17,6 +20,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -867,6 +871,302 @@ func TestReconcileMariaDBRestore_KindCheckedBeforeSrcBackupGet(t *testing.T) { } } +// The operator reports a failed backing Job as Complete=True with reason +// JobFailed (SetCompleteWithJob in mariadb-operator 25.10.2), so the +// Complete status alone does not tell success from failure. +var mariadbCompleteOutcomeCases = []struct { + name string + noCondition bool + status metav1.ConditionStatus + reason string + message string + expired bool + failedJob bool + wantFailed bool + wantDone bool + wantMessage []string +}{ + {name: "Complete=True/JobComplete succeeds", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success", wantDone: true}, + {name: "Complete=True/JobFailed fails with the Job's failure", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true, + wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}}, + {name: "Complete=False/JobFailed fails with the Job's failure", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true, + wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}}, + {name: "Complete=True/JobFailed with its Job gone fails", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", wantFailed: true, + wantMessage: []string{"tenant/op", "Job tenant/op not found"}}, + {name: "Complete=True with an unknown reason keeps waiting", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something"}, + {name: "Complete=False/JobComplete keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success"}, + {name: "Complete=False/JobRunning keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobRunning, message: "Running"}, + {name: "Complete=False/Failed keeps waiting", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job"}, + {name: "no Complete condition keeps waiting", noCondition: true}, + {name: "no Complete condition past the deadline fails", noCondition: true, expired: true, wantFailed: true, + wantMessage: []string{"no Complete condition observed"}}, + {name: "Complete=True with an unknown reason past the deadline fails", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something", expired: true, wantFailed: true, + wantMessage: []string{"Complete=True/SomethingElse"}}, + {name: "Complete=False/Failed past the deadline carries the operator's message", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job", expired: true, wantFailed: true, + wantMessage: []string{"Complete=False/Failed: Error creating Job"}}, +} + +func mariadbCompleteConditions(noCondition bool, status metav1.ConditionStatus, reason, message string) []metav1.Condition { + if noCondition { + return nil + } + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: metav1.Now(), + }} +} + +// mariadbOutcomeStartedAt returns a StartedAt that is fresh, or older than +// both the backup and the default restore deadline. +func mariadbOutcomeStartedAt(expired bool) *metav1.Time { + if !expired { + now := metav1.Now() + return &now + } + past := metav1.NewTime(time.Now().Add(-2 * max(mariadbDefaultBackupDeadline, mariadbDefaultRestoreDeadline))) + return &past +} + +// mariadbOutcomeObjects adds the operator's Job, named after the operator +// Backup or Restore, failed the way the Job controller fails it once the +// backoff limit is spent. +func mariadbOutcomeObjects(failedJob bool, objs ...client.Object) []client.Object { + if !failedJob { + return objs + } + return append(objs, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Reason: batchv1.JobReasonBackoffLimitExceeded, + Message: "Job has reached the specified backoff limit", + }}}, + }) +} + +func assertMariaDBFailureMessage(t *testing.T, message string, want []string) { + t.Helper() + if strings.Contains(message, "logs") { + t.Errorf("failure message must not send the reader to Job logs, which are gone with the deleted pod: %q", message) + } + for _, w := range want { + if !strings.Contains(message, w) { + t.Errorf("failure message must contain %q, got %q", w, message) + } + } +} + +func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) { + strategyAPIGroup := strategyv1alpha1.GroupVersion.Group + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyAPIGroup, + Kind: strategyv1alpha1.MariaDBStrategyKind, + Name: "mdb-strategy", + }, + } + + for _, tc := range mariadbCompleteOutcomeCases { + t.Run(tc.name, func(t *testing.T) { + seed := newMariaDBBackupJob("bj", "tenant") + seed.Status.StartedAt = mariadbOutcomeStartedAt(tc.expired) + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp(seed.Spec.ApplicationRef.Name)}, + } + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "op", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: seed.Name, + backupsv1alpha1.OwningJobNamespaceLabel: seed.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)}, + } + c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, seed, newMariaDBApp(seed.Spec.ApplicationRef.Name, "tenant"), + strategy, operatorMariaDB, operatorBackup)...) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} + + bj := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), bj); err != nil { + t.Fatalf("seed Get: %v", err) + } + res, err := r.reconcileMariaDB(context.Background(), bj, resolved) + if err != nil { + t.Fatalf("reconcileMariaDB: %v", err) + } + + stored := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), stored); err != nil { + t.Fatalf("persisted Get: %v", err) + } + artifacts := &backupsv1alpha1.BackupList{} + if err := c.List(context.Background(), artifacts, client.InNamespace("tenant")); err != nil { + t.Fatalf("list Backups: %v", err) + } + switch { + case tc.wantFailed: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("expected Phase=Failed, got %q", stored.Status.Phase) + } + assertMariaDBFailureMessage(t, stored.Status.Message, tc.wantMessage) + if stored.Status.BackupRef != nil || len(artifacts.Items) != 0 { + t.Errorf("a failed run must not produce a Backup: ref=%v artifacts=%d", stored.Status.BackupRef, len(artifacts.Items)) + } + case tc.wantDone: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Errorf("expected Phase=Succeeded, got %q (message %q)", stored.Status.Phase, stored.Status.Message) + } + if stored.Status.BackupRef == nil || len(artifacts.Items) != 1 { + t.Errorf("a successful run must produce exactly one Backup: ref=%v artifacts=%d", stored.Status.BackupRef, len(artifacts.Items)) + } + default: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning { + t.Errorf("expected Phase=Running, got %q", stored.Status.Phase) + } + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the run is in progress, got %+v", res) + } + if len(artifacts.Items) != 0 { + t.Errorf("an unfinished run must not produce a Backup, got %d", len(artifacts.Items)) + } + } + }) + } +} + +func TestMariaDBJobFailedMessage_JobUnreadable(t *testing.T) { + cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed} + readErr := apierrors.NewForbidden(batchv1.Resource("jobs"), "op", errors.New("rbac")) + c := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*batchv1.Job); ok { + return readErr + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + + got := mariadbJobFailedMessage(context.Background(), c, "Backup", "tenant", "op", cond) + for _, want := range []string{"k8s.mariadb.com/Backup tenant/op failed (Complete=True/JobFailed)", "reading Job tenant/op", readErr.Error()} { + if !strings.Contains(got, want) { + t.Errorf("message must contain %q, got %q", want, got) + } + } +} + +func TestMariaDBJobFailedMessage_JobWithoutFailedCondition(t *testing.T) { + cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed} + c := newMariaDBStrategyTestClient(t, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue, Reason: batchv1.JobReasonPodFailurePolicy}, + {Type: batchv1.JobFailed, Status: corev1.ConditionFalse, Reason: batchv1.JobReasonBackoffLimitExceeded}, + }}, + }) + + got := mariadbJobFailedMessage(context.Background(), c, "Restore", "tenant", "op", cond) + if !strings.Contains(got, "Job tenant/op reports no Failed condition") || + strings.Contains(got, batchv1.JobReasonBackoffLimitExceeded) || strings.Contains(got, batchv1.JobReasonPodFailurePolicy) { + t.Errorf("a Failed condition that is not True must not be reported as the cause, got %q", got) + } +} + +func TestMariaDBCompleteDetail(t *testing.T) { + for _, tc := range []struct { + cond *metav1.Condition + want string + }{ + {nil, "no Complete condition observed"}, + {&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed", Message: "Backup not complete"}, "Complete=False/Failed: Backup not complete"}, + {&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed"}, "Complete=False/Failed"}, + } { + if got := mariadbCompleteDetail(tc.cond); got != tc.want { + t.Errorf("mariadbCompleteDetail(%+v) = %q, want %q", tc.cond, got, tc.want) + } + } +} + +func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) { + apps := mariadbapp.GroupName + + for _, tc := range mariadbCompleteOutcomeCases { + t.Run(tc.name, func(t *testing.T) { + cozyBackup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + DriverMetadata: map[string]string{mariadbBackupNameKey: "src-op-bk"}, + }, + } + seed := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: mariadbOutcomeStartedAt(tc.expired), Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + operatorRestore := &mariadbtypes.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "op", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: seed.Name, + backupsv1alpha1.OwningJobNamespaceLabel: seed.Namespace, + }, + }, + Status: mariadbtypes.RestoreStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)}, + } + c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, cozyBackup, seed, operatorRestore, + &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-op-bk"}}, + &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}})...) + r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), rj); err != nil { + t.Fatalf("seed Get: %v", err) + } + res, err := r.reconcileMariaDBRestore(context.Background(), rj, cozyBackup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + + stored := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), stored); err != nil { + t.Fatalf("persisted Get: %v", err) + } + switch { + case tc.wantFailed: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("expected Phase=Failed, got %q", stored.Status.Phase) + } + assertMariaDBFailureMessage(t, stored.Status.Message, tc.wantMessage) + case tc.wantDone: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Errorf("expected Phase=Succeeded, got %q (message %q)", stored.Status.Phase, stored.Status.Message) + } + default: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning { + t.Errorf("expected Phase=Running, got %q", stored.Status.Phase) + } + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the run is in progress, got %+v", res) + } + } + }) + } +} + // --------------------------------------------------------------------------- // Test helpers // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/mariadbtypes/types.go b/internal/backupcontroller/mariadbtypes/types.go index 7bc5b03a1..61fc71019 100644 --- a/internal/backupcontroller/mariadbtypes/types.go +++ b/internal/backupcontroller/mariadbtypes/types.go @@ -27,16 +27,20 @@ const ( GroupName = "k8s.mariadb.com" Version = "v1alpha1" - // Operator-side condition type set on Backup and Restore once the run - // terminates. Status=True means success; Status=False with Reason - // explains the failure or in-progress state. + // Operator-side condition type set on Backup and Restore. Status=True + // means the backing Job terminated, successfully or not; only the + // Reason tells the two apart (SetCompleteWithJob in mariadb-operator). ConditionTypeComplete = "Complete" // Reason values the upstream operator emits on the Complete condition. // JobComplete -> Status=True (run succeeded). - // JobFailed -> Status=False (terminal failure of the backing Job). + // JobFailed -> Status=True (terminal failure of the backing Job). // JobRunning -> Status=False (still in progress). // JobSuspended -> Status=False (admin-suspended via spec). + // The operator also writes reason Failed with Status=False and its own + // message ("Error creating Job", "Backup not complete") when a run fails + // outside its Job. It retries those, so the driver keeps waiting on + // Failed until its deadline rather than failing the run at once. ConditionReasonJobComplete = "JobComplete" ConditionReasonJobFailed = "JobFailed" ConditionReasonJobRunning = "JobRunning" @@ -183,7 +187,7 @@ type S3TLS struct { } // BackupStatus mirrors Backup.status. The operator expresses success -// via condition type "Complete" with status=True. +// via condition type "Complete" with reason JobComplete. type BackupStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } @@ -217,7 +221,7 @@ type RestoreSpec struct { } // RestoreStatus mirrors Restore.status. Same condition convention as -// Backup: type=Complete, Status=True on success. +// Backup: type=Complete, reason JobComplete on success. type RestoreStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go index 0ead1bc58..9caa218a7 100644 --- a/internal/backupcontroller/psmdbstrategy_controller.go +++ b/internal/backupcontroller/psmdbstrategy_controller.go @@ -1376,9 +1376,10 @@ func (r *BackupJobReconciler) createMongoDBBackupArtifact( // annotation (honoured before any apiserver read, so it still frees a Backup // whose operator CR has become unreadable) and a going-away namespace (the psmdb // delete-backup exec cannot run there) each make the driver strip its own -// finalizer and release, leaving the object for a bucket lifecycle policy. This -// bounds only what the driver contributes: a Get that fails with something other -// than NotFound/NoMatchError still requeues (below), and the psmdb operator's own +// finalizer and release, leaving the object to manual cleanup, or to a +// lifecycle rule on the bucket where the object store applies one. This bounds +// only what the driver contributes: a Get that fails with something other than +// NotFound/NoMatchError still requeues (below), and the psmdb operator's own // release-lock finalizer keeps the CR Terminating while the operator is down, // whatever the driver strips. A down operator in a live namespace is likewise // not auto-released: the Backup stays Terminating as a visible signal until an diff --git a/internal/backupcontroller/redis_artifact_size.go b/internal/backupcontroller/redis_artifact_size.go new file mode 100644 index 000000000..49a73d292 --- /dev/null +++ b/internal/backupcontroller/redis_artifact_size.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// redisArtifact locates the RDB snapshot the backup Job uploaded and returns +// an Artifact carrying its s3:// URI and size. The default Redis strategy +// takes its S3 coordinates from the projected backup credentials Secret, so +// the object is looked up there, with the TLS trust the Job's pod applied +// (see redisS3Trust). It returns an artifactProbeError, and no Artifact, +// when the Secret cannot be read or the object is not found: a custom +// strategy that writes elsewhere must not get a URI pointing at an object +// that does not exist. +func (r *BackupJobReconciler) redisArtifact(ctx context.Context, namespace, objectKey string, podSpec *corev1.PodSpec) (*backupsv1alpha1.BackupArtifact, error) { + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) + defer cancel() + + bucket, conn, err := r.projectedS3Connection(ctx, namespace) + if err != nil { + return nil, &artifactProbeError{reason: "the backup credentials Secret is unusable", err: err} + } + if conn.CAPEM, conn.InsecureSkipVerify, err = r.redisS3Trust(ctx, namespace, podSpec); err != nil { + return nil, &artifactProbeError{reason: "the S3 endpoint CA the Job mounted cannot be read", err: err} + } + // The script passes the CA as curl --cacert, which replaces the default + // trust store rather than adding to it. + conn.CAOnly = true + lister, err := r.s3Lister(conn) + if err != nil { + return nil, &artifactProbeError{reason: "the S3 endpoint is not usable", err: err} + } + size, err := lister.StatObject(ctx, bucket, objectKey) + if err != nil { + return nil, &artifactProbeError{reason: "the snapshot could not be found or read in the bucket", err: err} + } + return &backupsv1alpha1.BackupArtifact{ + URI: fmt.Sprintf("s3://%s/%s", bucket, objectKey), + SizeBytes: size, + }, nil +} + +// artifactProbeError carries a fixed reason fit for an Event in the +// application namespace; err may name the platform's endpoint and bucket, +// so it goes to the controller log only. +type artifactProbeError struct { + reason string + err error +} + +func (e *artifactProbeError) Error() string { return e.reason + ": " + e.err.Error() } +func (e *artifactProbeError) Unwrap() error { return e.err } + +// projectedS3Connection reads the backup credentials Secret the controller +// projects into the application namespace. The projector stores the endpoint +// without a scheme, and the strategy scripts reach it over https. It can +// still carry a path, which minio-go refuses. +func (r *BackupJobReconciler) projectedS3Connection(ctx context.Context, namespace string) (string, s3Connection, error) { + var conn s3Connection + name := r.CredentialsConfig.TargetSecretName + if name == "" { + return "", conn, errors.New("no projected credentials Secret is configured") + } + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", conn, err + } + value := func(key string) string { return string(secret.Data[key]) } + + bucket := value("bucketName") + conn.AccessKey, conn.SecretKey = value("AWS_ACCESS_KEY_ID"), value("AWS_SECRET_ACCESS_KEY") + if bucket == "" || value("endpoint") == "" || conn.AccessKey == "" || conn.SecretKey == "" { + return "", conn, fmt.Errorf("secret %s/%s lacks bucketName, endpoint or access keys", namespace, name) + } + conn.Endpoint, conn.Secure = value("endpoint"), true + conn.Region = value("region") + return bucket, conn, nil +} + +const ( + redisS3CAVolume = "s3-ca" + redisS3InsecureSkipEnv = "S3_INSECURE_SKIP_VERIFY" +) + +// redisS3Trust mirrors how the default strategy script verifies the S3 +// endpoint: it trusts ca.crt, then tls.crt, from the s3-ca volume, and only +// when neither exists does it honour S3_INSECURE_SKIP_VERIFY=true. Both are +// read from the pod spec the backup Job ran with, but the Secret is read +// now, so an s3-ca volume that yields no CA now - a Secret that is gone or +// lacks the keys, or a source other than a Secret - is refused: the Job may +// have verified against a CA that is gone, and neither the system roots nor +// skip-verify would trust as narrowly. The default strategy mounts s3-ca +// only when the admin configured an endpoint CA. +func (r *BackupJobReconciler) redisS3Trust(ctx context.Context, namespace string, podSpec *corev1.PodSpec) ([]byte, bool, error) { + if podSpec == nil { + return nil, false, nil + } + caUnknown := false + for _, v := range podSpec.Volumes { + if v.Name != redisS3CAVolume { + continue + } + if v.Secret == nil { + caUnknown = true + break + } + secret := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: v.Secret.SecretName}, secret) + if apierrors.IsNotFound(err) { + caUnknown = true + break + } + if err != nil { + return nil, false, err + } + for _, file := range []string{"ca.crt", "tls.crt"} { + if pem := secret.Data[secretKeyForFile(v.Secret.Items, file)]; len(pem) > 0 { + return pem, false, nil + } + } + caUnknown = true + } + if caUnknown { + return nil, false, fmt.Errorf("the %s volume yields no CA now", redisS3CAVolume) + } + return nil, redisSkipsVerify(podSpec.Containers), nil +} + +// redisSkipsVerify reports whether every container sets +// S3_INSECURE_SKIP_VERIFY=true: the probe cannot tell which container +// uploaded, so one that verified is enough to keep verification on. When a +// container lists the variable twice, the last entry wins, as in the +// kubelet. +func redisSkipsVerify(containers []corev1.Container) bool { + for _, c := range containers { + value := "" + for _, e := range c.Env { + if e.Name == redisS3InsecureSkipEnv { + value = e.Value + } + } + if value != "true" { + return false + } + } + return len(containers) > 0 +} + +// secretKeyForFile returns the Secret key a secret volume projects to file: +// the key itself when the volume lists no items. +func secretKeyForFile(items []corev1.KeyToPath, file string) string { + if len(items) == 0 { + return file + } + for _, it := range items { + if it.Path == file { + return it.Key + } + } + return "" +} diff --git a/internal/backupcontroller/redis_artifact_size_test.go b/internal/backupcontroller/redis_artifact_size_test.go new file mode 100644 index 000000000..47d357c4e --- /dev/null +++ b/internal/backupcontroller/redis_artifact_size_test.go @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/pem" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +const redisTestKey = "tenant/cache/bj-1.rdb" + +func projectedCredsSecret(data map[string]string) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "cozy-backups-creds"}, + Data: map[string][]byte{}, + } + for k, v := range map[string]string{ + "endpoint": "s3.example.org", + "bucketName": "cozy-backups", + "region": "us-east-1", + "AWS_ACCESS_KEY_ID": "AKID", + "AWS_SECRET_ACCESS_KEY": "secret", + } { + s.Data[k] = []byte(v) + } + for k, v := range data { + if v == "" { + delete(s.Data, k) + continue + } + s.Data[k] = []byte(v) + } + return s +} + +func redisReconciler(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler { + r := reconcilerWithLister(t, l, objs...) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + return r +} + +func TestRedisArtifact(t *testing.T) { + t.Run("stats the snapshot with the projected credentials", func(t *testing.T) { + l := &fakeS3Lister{statSize: 2048} + r := redisReconciler(t, l, projectedCredsSecret(nil)) + got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil) + if got == nil || got.URI != "s3://cozy-backups/"+redisTestKey || got.SizeBytes != 2048 { + t.Fatalf("artifact = %#v", got) + } + if l.bucket != "cozy-backups" || l.statKey != redisTestKey { + t.Errorf("stat %q/%q, want cozy-backups/%q", l.bucket, l.statKey, redisTestKey) + } + if l.conn.Endpoint != "s3.example.org" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.Region != "us-east-1" { + t.Errorf("connection = %+v", l.conn) + } + }) + + unset := map[string]struct { + lister *fakeS3Lister + objs []client.Object + noTarget bool + }{ + "missing object": {lister: &fakeS3Lister{statErr: errors.New("NoSuchKey")}, objs: []client.Object{projectedCredsSecret(nil)}}, + "missing Secret": {lister: &fakeS3Lister{statSize: 1}}, + "missing bucket": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"bucketName": ""})}}, + "missing keys": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"AWS_SECRET_ACCESS_KEY": ""})}}, + "no target name": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(nil)}, noTarget: true}, + "missing endpoint": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"endpoint": ""})}}, + "missing access key": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"AWS_ACCESS_KEY_ID": ""})}}, + } + for name, tc := range unset { + t.Run(name+" leaves the artifact unset", func(t *testing.T) { + r := redisReconciler(t, tc.lister, tc.objs...) + if tc.noTarget { + r.CredentialsConfig.TargetSecretName = "" + } + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil); got != nil { + t.Errorf("artifact = %#v, want nil", got) + } + }) + } +} + +// TestRedisArtifact_MinioClient runs the real minio client against a TLS +// stub with a self-signed certificate that answers the HEAD request. It +// covers, in one pass, the trust settings the backup Job used, request +// signing and Content-Length parsing. +func TestRedisArtifact_MinioClient(t *testing.T) { + // minio-go retries a TLS failure with backoff; a refusal needs no more + // than the first attempt. + saved := artifactSizeProbeTimeout + artifactSizeProbeTimeout = time.Second + defer func() { artifactSizeProbeTimeout = saved }() + var gotMethod, gotPath, gotAuth string + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotMethod, gotPath, gotAuth = req.Method, req.URL.Path, req.Header.Get("Authorization") + w.Header().Set("Content-Length", "4096") + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + + creds := projectedCredsSecret(map[string]string{"endpoint": strings.TrimPrefix(srv.URL, "https://")}) + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": caPEM}, + } + c := newMariaDBStrategyTestClient(t, creds, ca) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, &corev1.PodSpec{}); got != nil { + t.Errorf("without the Job's trust settings: artifact = %#v, want nil (untrusted certificate)", got) + } + + got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "")) + if got == nil || got.SizeBytes != 4096 { + t.Fatalf("with the s3-ca volume: artifact = %#v, want SizeBytes 4096", got) + } + if gotMethod != http.MethodHead || gotPath != "/cozy-backups/"+redisTestKey { + t.Errorf("request %s %s, want HEAD /cozy-backups/%s", gotMethod, gotPath, redisTestKey) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } + + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("", "true")); got == nil || got.SizeBytes != 4096 { + t.Errorf("with S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want SizeBytes 4096", got) + } + + ca.Data["ca.crt"] = unrelatedCAPEM(t) + if err := c.Update(context.Background(), ca); err != nil { + t.Fatalf("update CA Secret: %v", err) + } + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "true")); got != nil { + t.Errorf("with a CA that did not sign the endpoint and S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want nil", got) + } +} + +// redisPodSpec builds the parts of the backup Job's pod spec that +// redisS3Trust reads: the s3-ca volume when caSecret is set, and the +// S3_INSECURE_SKIP_VERIFY variable. +func redisPodSpec(caSecret, skipVerify string) *corev1.PodSpec { + spec := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: "backup", + Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: skipVerify}}, + }}} + if caSecret != "" { + spec.Volumes = []corev1.Volume{{ + Name: "s3-ca", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: caSecret}}, + }} + } + return spec +} + +func withVolume(spec *corev1.PodSpec, name, secret string) *corev1.PodSpec { + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: secret}}, + }) + return spec +} + +func configMapS3CA(spec *corev1.PodSpec) *corev1.PodSpec { + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "s3-ca", + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "s3-endpoint-ca"}}}, + }) + return spec +} + +// nextEvent returns the next recorded event, or "" when there is none. +func nextEvent(rec *record.FakeRecorder) string { + select { + case ev := <-rec.Events: + return ev + default: + return "" + } +} + +func TestRedisS3Trust(t *testing.T) { + caSecret := func(data map[string]string) *corev1.Secret { + s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, Data: map[string][]byte{}} + for k, v := range data { + s.Data[k] = []byte(v) + } + return s + } + cases := map[string]struct { + spec *corev1.PodSpec + objs []client.Object + wantCA string + wantSkip bool + }{ + "ca.crt wins over tls.crt and skip-verify": { + spec: redisPodSpec("s3-endpoint-ca", "true"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA", "tls.crt": "TLS"})}, + wantCA: "CA", + }, + "tls.crt when there is no ca.crt": { + spec: redisPodSpec("s3-endpoint-ca", ""), + objs: []client.Object{caSecret(map[string]string{"tls.crt": "TLS"})}, + wantCA: "TLS", + }, + "volume items map the file to another key": { + spec: func() *corev1.PodSpec { + s := redisPodSpec("s3-endpoint-ca", "") + s.Volumes[0].Secret.Items = []corev1.KeyToPath{{Key: "bundle.pem", Path: "ca.crt"}} + return s + }(), + objs: []client.Object{caSecret(map[string]string{"bundle.pem": "MAPPED", "ca.crt": "UNMAPPED"})}, + wantCA: "MAPPED", + }, + "skip-verify without a CA volume": { + spec: redisPodSpec("", "true"), + wantSkip: true, + }, + "skip-verify only when it is exactly true": { + spec: redisPodSpec("", "false"), + }, + "no pod spec": {}, + "the Redis CA volume is not the S3 CA": { + spec: withVolume(redisPodSpec("", "true"), "redis-ca", "redis-tls"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}}, + wantSkip: true, + }, + "the S3 CA next to the Redis CA": { + spec: withVolume(withVolume(&corev1.PodSpec{Containers: redisPodSpec("", "").Containers}, "redis-ca", "redis-tls"), "s3-ca", "s3-endpoint-ca"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}}, + wantCA: "CA", + }, + "another variable set to true is not skip-verify": { + spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{{Name: "REDIS_TLS", Value: "true"}}}}}, + }, + "skip-verify only when every container skips": { + spec: &corev1.PodSpec{Containers: []corev1.Container{ + {Name: "upload", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"}}}, + {Name: "sidecar", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"}}}, + }}, + }, + "the last entry of a repeated variable wins": { + spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{ + {Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"}, + {Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"}, + }}}}, + }, + } + // Each s3-ca volume that yields no CA now is refused, with or without + // skip-verify: the Job may have verified against a CA that is gone. + refused := map[string]struct { + spec func(skip string) *corev1.PodSpec + objs []client.Object + }{ + "a CA Secret that is gone": {spec: func(skip string) *corev1.PodSpec { return redisPodSpec("s3-endpoint-ca", skip) }}, + "a CA Secret without CA keys": { + spec: func(skip string) *corev1.PodSpec { return redisPodSpec("s3-endpoint-ca", skip) }, + objs: []client.Object{caSecret(map[string]string{"other": "x"})}, + }, + "an s3-ca volume of another source": {spec: func(skip string) *corev1.PodSpec { return configMapS3CA(redisPodSpec("", skip)) }}, + "a CA Secret whose keys the volume does not project": { + spec: func(skip string) *corev1.PodSpec { + s := redisPodSpec("s3-endpoint-ca", skip) + s.Volumes[0].Secret.Items = []corev1.KeyToPath{{Key: "bundle.pem", Path: "ca.crt"}} + return s + }, + objs: []client.Object{caSecret(map[string]string{"tls.crt": "UNPROJECTED"})}, + }, + } + for name, tc := range refused { + for _, skip := range []string{"", "true"} { + t.Run(name+" is refused, skip="+skip, func(t *testing.T) { + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, tc.objs...)} + ca, gotSkip, err := r.redisS3Trust(context.Background(), "tenant", tc.spec(skip)) + if err == nil || gotSkip || ca != nil { + t.Errorf("trust = (%q, %v, %v), want an error, no CA and no skip-verify", ca, gotSkip, err) + } + }) + } + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := newMariaDBStrategyTestClient(t, tc.objs...) + r := &BackupJobReconciler{Client: c} + ca, skip, err := r.redisS3Trust(context.Background(), "tenant", tc.spec) + if err != nil { + t.Fatalf("redisS3Trust: %v", err) + } + if string(ca) != tc.wantCA || skip != tc.wantSkip { + t.Errorf("trust = (%q, %v), want (%q, %v)", ca, skip, tc.wantCA, tc.wantSkip) + } + }) + } +} + +func TestCreateRedisBackupArtifact_Artifact(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: "Redis", Name: "cache"}, + }, + } + + t.Run("records the snapshot location and size", func(t *testing.T) { + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 || got.Status.Artifact.URI != "s3://cozy-backups/"+redisTestKey { + t.Fatalf("Artifact = %#v", got.Status.Artifact) + } + }) + + t.Run("an unmeasured artifact is reported on the BackupJob without the raw error", func(t *testing.T) { + l := &fakeS3Lister{statErr: errors.New("Get https://s3.internal.example/cozy-backups: NoSuchKey")} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Status.Artifact != nil { + t.Fatalf("createRedisBackupArtifact = %#v, %v; want a Backup without an artifact", got, err) + } + select { + case ev := <-rec.Events: + if !strings.HasPrefix(ev, "Normal ArtifactNotMeasured ") || !strings.Contains(ev, "could not be found or read") { + t.Errorf("event = %q, want Normal ArtifactNotMeasured with the reason", ev) + } + if strings.Contains(ev, "s3.internal.example") { + t.Errorf("event = %q leaks the raw S3 error into the application namespace", ev) + } + default: + t.Error("no event on the BackupJob for an artifact left unset") + } + }) + + t.Run("a measured artifact raises no event", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statSize: 1}, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if len(rec.Events) != 0 { + t.Errorf("event %q for an artifact that was recorded", <-rec.Events) + } + }) + + t.Run("an existing Backup is returned without probing or creating", func(t *testing.T) { + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, existing, projectedCredsSecret(nil)) + creates := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + creates++ + return cl.Create(ctx, obj, opts...) + }, + }) + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times for a Backup that already exists", l.calls) + } + if creates != 0 { + t.Errorf("Create called %d times for a Backup that already exists", creates) + } + }) + + t.Run("an empty snapshot is recorded and warned about", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statSize: 0}, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Status.Artifact == nil || got.Status.Artifact.URI != "s3://cozy-backups/"+redisTestKey { + t.Fatalf("createRedisBackupArtifact = %#v, %v; want the artifact recorded", got, err) + } + if ev := nextEvent(rec); !strings.HasPrefix(ev, "Warning EmptySnapshot ") { + t.Errorf("event = %q, want Warning EmptySnapshot", ev) + } + }) +} + +// TestReconcileRedis_ArtifactUsesTheJobTrust pins the wiring from the +// completed backup Job to the probe: the s3-ca volume of the pod the Job +// ran must reach the S3 connection. +func TestReconcileRedis_ArtifactUsesTheJobTrust(t *testing.T) { + app := newRedisApp("cache", "tenant-test") + strategy := newRedisStrategy("redis-strategy") + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newRedisAppRef("cache"), + BackupClassName: "cozy-default", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobNameForBackupJob(backupJob), + Namespace: backupJob.Namespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: backupJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace, + }, + }, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: *redisPodSpec("s3-endpoint-ca", "")}}, + Status: batchv1.JobStatus{ + CompletionTime: &now, + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + creds := projectedCredsSecret(nil) + creds.Namespace = "tenant-test" + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": []byte("CA")}, + } + resolved := newRedisResolved("redis-strategy", map[string]string{"bucketName": "redis-bucket"}) + + r, _ := newRedisTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob, creds, ca)) + l := &fakeS3Lister{statSize: 512} + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + r.newS3ObjectLister = func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + } + ctx := context.Background() + + if _, err := r.reconcileRedis(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileRedis() error = %v", err) + } + if string(l.conn.CAPEM) != "CA" { + t.Errorf("connection CA = %q, want the s3-ca volume's ca.crt", l.conn.CAPEM) + } + + updated := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil { + t.Fatalf("get backupjob: %v", err) + } + if updated.Status.BackupRef == nil { + t.Fatalf("expected BackupRef set") + } + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: updated.Status.BackupRef.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if created.Status.Artifact == nil || created.Status.Artifact.SizeBytes != 512 { + t.Errorf("Artifact = %#v, want SizeBytes 512", created.Status.Artifact) + } +} + +func TestCreateRedisBackupArtifact_RetryCheck(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: "Redis", Name: "cache"}, + }, + } + + t.Run("a failed existence check creates the Backup without probing", func(t *testing.T) { + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil { + t.Fatalf("createRedisBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } + if ev := nextEvent(rec); !strings.HasPrefix(ev, "Normal ArtifactNotMeasured ") || !strings.Contains(ev, "could not be checked") { + t.Errorf("event = %q, want ArtifactNotMeasured saying the existence check failed", ev) + } + }) + + t.Run("no event when Create fails", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statErr: errors.New("NoSuchKey")}, j, projectedCredsSecret(nil)) + r.Client = forbidBackupCreate(t, r.Client) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err == nil { + t.Fatal("createRedisBackupArtifact succeeded although Create was rejected") + } + if ev := nextEvent(rec); ev != "" { + t.Errorf("event %q although the BackupJob fails", ev) + } + }) + + t.Run("no event when Create collides with a Backup another pass made", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statErr: errors.New("NoSuchKey")}) + r.Client, r.APIReader = staleCache(t, store), notFoundOnce(t, store) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if ev := nextEvent(rec); ev != "" { + t.Errorf("event %q about a Backup this pass did not create", ev) + } + }) + + t.Run("a Create collision is resolved from the API server past the cache", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statSize: 777}) + r.Client, r.APIReader = staleCache(t, store), notFoundOnce(t, store) + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Name != "bj-1" { + t.Fatalf("createRedisBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } + }) + + t.Run("an existing Backup that cannot be read back is retried", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statSize: 777}) + r.Client, r.APIReader = staleCache(t, store), failingReads(t, store) + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } + }) +} + +func TestRedisArtifact_HangingEndpointIsGivenUp(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{}, projectedCredsSecret(nil)) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil); got != nil { + t.Errorf("artifact = %#v, want nil", got) + } + }) +} + +func TestRedisArtifact_TrustsOnlyTheCA(t *testing.T) { + l := &fakeS3Lister{statSize: 1} + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + r := redisReconciler(t, l, projectedCredsSecret(nil), ca) + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "")); got == nil { + t.Fatal("artifact = nil") + } + if !l.conn.CAOnly { + t.Error("the probe trusts the system roots on top of the CA, which curl --cacert does not") + } +} + +// TestReconcileRedis_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileRedis_UnreadableBackupIsRetried(t *testing.T) { + app := newRedisApp("cache", "tenant-test") + strategy := newRedisStrategy("redis-strategy") + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newRedisAppRef("cache"), + BackupClassName: "cozy-default", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobNameForBackupJob(backupJob), + Namespace: backupJob.Namespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: backupJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace, + }, + }, + Status: batchv1.JobStatus{ + CompletionTime: &now, + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: "test-bj"}} + resolved := newRedisResolved("redis-strategy", map[string]string{"bucketName": "redis-bucket"}) + + r, _ := newRedisTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob, existing)) + store := r.Client + r.Client, r.APIReader = staleCache(t, store), failingReads(t, store) + ctx := context.Background() + + if _, err := r.reconcileRedis(ctx, backupJob, resolved); !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileRedis err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(ctx, client.ObjectKeyFromObject(backupJob), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} diff --git a/internal/backupcontroller/redisstrategy_controller.go b/internal/backupcontroller/redisstrategy_controller.go index 7d287d385..4169e6cef 100644 --- a/internal/backupcontroller/redisstrategy_controller.go +++ b/internal/backupcontroller/redisstrategy_controller.go @@ -3,6 +3,7 @@ package backupcontroller import ( "context" + "errors" "fmt" "maps" "strings" @@ -366,9 +367,9 @@ func (r *BackupJobReconciler) reconcileRedis(ctx context.Context, j *backupsv1al if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createRedisBackupArtifact(ctx, j, resolved, objectKey) + artifact, err := r.createRedisBackupArtifact(ctx, j, resolved, objectKey, &batchJob.Spec.Template.Spec) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -471,7 +472,12 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig, objectKey string, + podSpec *corev1.PodSpec, ) (*backupsv1alpha1.Backup, error) { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { + return existing, nil + } driverMD := map[string]string{redisObjectMetaKey: objectKey} for k, v := range resolved.Parameters { driverMD[redisParamPrefix+k] = v @@ -492,6 +498,19 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( Phase: backupsv1alpha1.BackupPhaseReady, }, } + unmeasured := "whether the Backup already exists could not be checked" + if probe { + artifact, err := r.redisArtifact(ctx, j.Namespace, objectKey, podSpec) + unmeasured = "" + if err != nil { + getLogger(ctx).Info("leaving the Redis backup artifact unset", "backupjob", j.Name, "objectKey", objectKey, "error", err) + var probeErr *artifactProbeError + if errors.As(err, &probeErr) { + unmeasured = probeErr.reason + } + } + backup.Status.Artifact = artifact + } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef } @@ -500,11 +519,23 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } + // Only once the Backup exists: a terminal Create error fails the + // BackupJob, and a collision returns a Backup another pass measured. + if r.Recorder != nil { + switch { + case unmeasured != "": + r.Recorder.Eventf(j, corev1.EventTypeNormal, "ArtifactNotMeasured", "The backup succeeded, but its location and size were not recorded: %s", unmeasured) + case backup.Status.Artifact != nil && backup.Status.Artifact.SizeBytes == 0: + // sizeBytes is omitempty, so a 0 reads as "not measured" on the + // Backup; a valid RDB snapshot always carries a header. + r.Recorder.Event(j, corev1.EventTypeWarning, "EmptySnapshot", "The backup's RDB snapshot is empty (0 bytes); a valid snapshot is never empty, so this backup is likely broken.") + } + } return backup, nil } diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..1a4dd7943 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -204,6 +204,8 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.reconcileRedisRestore(ctx, restoreJob, backup) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafkaRestore(ctx, restoreJob, backup) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetesRestore(ctx, restoreJob, backup) default: return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind)) } @@ -382,6 +384,12 @@ func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob * case strategyv1alpha1.VeleroStrategyKind: r.cleanupVeleroRestore(ctx, restoreJob) + case strategyv1alpha1.KubernetesStrategyKind: + // Its Velero Restores are labelled like the Velero driver's, so the + // same cleanup reaps them; a live cluster it stopped is resumed. + r.cleanupVeleroRestore(ctx, restoreJob) + r.cleanupKubernetesRestore(ctx, restoreJob) + case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind: // Nothing to clean up: these drivers don't materialise namespaced // artifacts that outlive the RestoreJob. (Etcd: the operator-side diff --git a/internal/observabilityproxy/auth.go b/internal/observabilityproxy/auth.go new file mode 100644 index 000000000..0baac558a --- /dev/null +++ b/internal/observabilityproxy/auth.go @@ -0,0 +1,172 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// ErrUnauthenticated means the token is missing or kube-apiserver rejected it. +var ErrUnauthenticated = errors.New("unauthenticated") + +// Authorizer asks kube-apiserver who a token belongs to and whether that +// identity may read a scope. Answers, including negative ones, are cached +// for the TTL; transport errors and reviews that failed to decide are not, +// so an outage is retried on the next request instead of locking a user +// out for the TTL. +type Authorizer struct { + client kubernetes.Interface + now func() time.Time + tokens *ttlCache[*authenticationv1.UserInfo] + decisions *ttlCache[bool] +} + +func NewAuthorizer(client kubernetes.Interface, ttl time.Duration) *Authorizer { + return &Authorizer{ + client: client, + now: time.Now, + tokens: newTTLCache[*authenticationv1.UserInfo](ttl), + decisions: newTTLCache[bool](ttl), + } +} + +func (a *Authorizer) Authenticate(ctx context.Context, token string) (*authenticationv1.UserInfo, error) { + if token == "" { + return nil, ErrUnauthenticated + } + sum := sha256.Sum256([]byte(token)) + key := hex.EncodeToString(sum[:]) + if u, ok := a.tokens.get(key, a.now()); ok { + if u == nil { + return nil, ErrUnauthenticated + } + return u, nil + } + tr, err := a.client.AuthenticationV1().TokenReviews().Create(ctx, + &authenticationv1.TokenReview{Spec: authenticationv1.TokenReviewSpec{Token: token}}, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("token review: %w", err) + } + if !tr.Status.Authenticated || tr.Status.User.Username == "" { + // status.error also marks an authenticator that failed to decide; + // a cached refusal would outlast its recovery. + if tr.Status.Error == "" { + a.tokens.set(key, nil, a.now()) + } + return nil, ErrUnauthenticated + } + u := tr.Status.User.DeepCopy() + a.tokens.set(key, u, a.now()) + return u, nil +} + +func (a *Authorizer) Authorize(ctx context.Context, user *authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes) (bool, error) { + extra := make(map[string]authorizationv1.ExtraValue, len(user.Extra)) + for k, v := range user.Extra { + extra[k] = authorizationv1.ExtraValue(v) + } + spec := authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &attrs, + User: user.Username, + UID: user.UID, + Groups: user.Groups, + Extra: extra, + } + raw, err := json.Marshal(spec) + if err != nil { + return false, err + } + key := string(raw) + if allowed, ok := a.decisions.get(key, a.now()); ok { + return allowed, nil + } + sar, err := a.client.AuthorizationV1().SubjectAccessReviews().Create(ctx, + &authorizationv1.SubjectAccessReview{Spec: spec}, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Errorf("subject access review: %w", err) + } + allowed := sar.Status.Allowed && !sar.Status.Denied + // An evaluation error without a decision denies this request only: it + // may be a webhook authorizer that is down, and RBAC reports one for + // any binding to a missing role, so it cannot be told from a refusal. + if allowed || sar.Status.Denied || sar.Status.EvaluationError == "" { + a.decisions.set(key, allowed, a.now()) + } + return allowed, nil +} + +// ttlCacheMaxEntries bounds memory against a flood of distinct tokens; once +// reached, expired entries are swept and, if that is not enough, the cache +// starts over, which costs only extra reviews. +const ttlCacheMaxEntries = 10000 + +type ttlEntry[V any] struct { + value V + expires time.Time +} + +type ttlCache[V any] struct { + mu sync.Mutex + ttl time.Duration + entries map[string]ttlEntry[V] +} + +func newTTLCache[V any](ttl time.Duration) *ttlCache[V] { + return &ttlCache[V]{ttl: ttl, entries: map[string]ttlEntry[V]{}} +} + +func (c *ttlCache[V]) get(key string, now time.Time) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok || !now.Before(e.expires) { + var zero V + return zero, false + } + return e.value, true +} + +func (c *ttlCache[V]) set(key string, v V, now time.Time) { + if c.ttl <= 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.entries) >= ttlCacheMaxEntries { + for k, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, k) + } + } + if len(c.entries) >= ttlCacheMaxEntries { + c.entries = map[string]ttlEntry[V]{} + } + } + c.entries[key] = ttlEntry[V]{value: v, expires: now.Add(c.ttl)} +} diff --git a/internal/observabilityproxy/auth_test.go b/internal/observabilityproxy/auth_test.go new file mode 100644 index 000000000..fe940ce04 --- /dev/null +++ b/internal/observabilityproxy/auth_test.go @@ -0,0 +1,220 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// fakeKube answers TokenReviews for the tokens in users and allows a +// SubjectAccessReview when allow returns true; it counts both calls. +type fakeKube struct { + client *fake.Clientset + reviews atomic.Int32 + sars atomic.Int32 + lastSAR atomic.Pointer[authorizationv1.SubjectAccessReviewSpec] + reviewFn func() error + // reviewError and evaluationError, when set, are what the + // authenticator or the authorizer report as failing to decide. + reviewError atomic.Pointer[string] + evaluationError atomic.Pointer[string] +} + +func newFakeKube(users map[string]string, allow func(user string, attrs *authorizationv1.ResourceAttributes) bool) *fakeKube { + f := &fakeKube{client: fake.NewClientset()} + f.client.PrependReactor("create", "tokenreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + f.reviews.Add(1) + if f.reviewFn != nil { + if err := f.reviewFn(); err != nil { + return true, nil, err + } + } + tr := action.(k8stesting.CreateAction).GetObject().(*authenticationv1.TokenReview).DeepCopy() + if e := f.reviewError.Load(); e != nil { + tr.Status = authenticationv1.TokenReviewStatus{Error: *e} + } else if name, ok := users[tr.Spec.Token]; ok { + tr.Status = authenticationv1.TokenReviewStatus{Authenticated: true, User: authenticationv1.UserInfo{Username: name, Groups: []string{"tenant-a-view"}}} + } + return true, tr, nil + }) + f.client.PrependReactor("create", "subjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + f.sars.Add(1) + sar := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SubjectAccessReview).DeepCopy() + spec := sar.Spec + f.lastSAR.Store(&spec) + if e := f.evaluationError.Load(); e != nil { + sar.Status.EvaluationError = *e + return true, sar, nil + } + sar.Status.Allowed = allow(sar.Spec.User, sar.Spec.ResourceAttributes) + return true, sar, nil + }) + return f +} + +func allowNamespace(ns string) func(string, *authorizationv1.ResourceAttributes) bool { + return func(_ string, a *authorizationv1.ResourceAttributes) bool { return a != nil && a.Namespace == ns } +} + +func TestAuthenticate(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + + u, err := a.Authenticate(ctx, "good") + if err != nil || u.Username != "alice" { + t.Fatalf("Authenticate(good) = %+v, %v", u, err) + } + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + if n := kube.reviews.Load(); n != 1 { + t.Fatalf("TokenReviews = %d, want 1 (second call cached)", n) + } + if _, err := a.Authenticate(ctx, "bad"); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("Authenticate(bad) err = %v, want ErrUnauthenticated", err) + } + if _, err := a.Authenticate(ctx, ""); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("Authenticate(\"\") err = %v, want ErrUnauthenticated", err) + } +} + +func TestAuthenticateCacheExpires(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + now := time.Unix(1000, 0) + a.now = func() time.Time { return now } + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + now = now.Add(11 * time.Second) + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + if n := kube.reviews.Load(); n != 2 { + t.Fatalf("TokenReviews = %d, want 2 after TTL", n) + } +} + +func TestAuthenticateErrorNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + kube.reviewFn = func() error { return errors.New("apiserver down") } + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); err == nil || errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want a transport error", err) + } + kube.reviewFn = nil + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatalf("err = %v after recovery", err) + } +} + +func TestAuthorize(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + user := &authenticationv1.UserInfo{Username: "alice", UID: "u1", Groups: []string{"g"}, Extra: map[string]authenticationv1.ExtraValue{"scope": {"x"}}} + + allowed := Route{Kind: NamespaceLogs, Namespace: "tenant-a"} + denied := Route{Kind: NamespaceLogs, Namespace: "tenant-b"} + + ok, err := a.Authorize(ctx, user, allowed.ResourceAttributes()) + if err != nil || !ok { + t.Fatalf("allow: %v, %v", ok, err) + } + spec := kube.lastSAR.Load() + if spec.User != "alice" || spec.UID != "u1" || len(spec.Groups) != 1 || spec.Extra["scope"][0] != "x" { + t.Fatalf("SAR identity not forwarded: %+v", spec) + } + if ra := spec.ResourceAttributes; ra.Verb != "list" || ra.Group != "cozystack.io" || ra.Resource != "logs" || ra.Namespace != "tenant-a" { + t.Fatalf("SAR attributes = %+v", ra) + } + + ok, err = a.Authorize(ctx, user, denied.ResourceAttributes()) + if err != nil || ok { + t.Fatalf("deny: %v, %v", ok, err) + } + + for range 3 { + if ok, _ := a.Authorize(ctx, user, allowed.ResourceAttributes()); !ok { + t.Fatal("cached allow flipped") + } + if ok, _ := a.Authorize(ctx, user, denied.ResourceAttributes()); ok { + t.Fatal("cached deny flipped") + } + } + if n := kube.sars.Load(); n != 2 { + t.Fatalf("SubjectAccessReviews = %d, want 2 (rest cached)", n) + } + + other := &authenticationv1.UserInfo{Username: "alice", UID: "u1", Groups: []string{"g", "tenant-b-view"}} + if _, err := a.Authorize(ctx, other, denied.ResourceAttributes()); err != nil { + t.Fatal(err) + } + if n := kube.sars.Load(); n != 3 { + t.Fatalf("a different group set must not hit another identity's cache entry, SARs = %d", n) + } +} + +// TestAuthenticateFailedReviewNotCached: a TokenReview the authenticator +// could not decide carries status.error; caching it would refuse a valid +// token for the TTL after the authenticator recovers. +func TestAuthenticateFailedReviewNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + outage := "webhook authenticator unreachable" + kube.reviewError.Store(&outage) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + kube.reviewError.Store(nil) + if u, err := a.Authenticate(ctx, "good"); err != nil || u.Username != "alice" { + t.Fatalf("after recovery: %+v, %v", u, err) + } +} + +// TestAuthorizeEvaluationErrorNotCached: a SubjectAccessReview with an +// evaluation error and no decision is a denial for this request only. +func TestAuthorizeEvaluationErrorNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + failure := "webhook authorizer unreachable" + kube.evaluationError.Store(&failure) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + user := &authenticationv1.UserInfo{Username: "alice"} + attrs := Route{Kind: NamespaceLogs, Namespace: "tenant-a"}.ResourceAttributes() + if ok, err := a.Authorize(ctx, user, attrs); err != nil || ok { + t.Fatalf("undecided review = %v, %v; want a denial", ok, err) + } + kube.evaluationError.Store(nil) + if ok, err := a.Authorize(ctx, user, attrs); err != nil || !ok { + t.Fatalf("after recovery: %v, %v", ok, err) + } +} diff --git a/internal/observabilityproxy/limits.go b/internal/observabilityproxy/limits.go new file mode 100644 index 000000000..06cc41d41 --- /dev/null +++ b/internal/observabilityproxy/limits.go @@ -0,0 +1,166 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "sync" + "time" +) + +// LimiterConfig holds the per-user and total caps. A zero cap disables it. +type LimiterConfig struct { + MaxConcurrentPerUser int + RatePerUser float64 + RateBurst int + MaxTailsPerUser int + MaxTails int +} + +// limiter keeps open tails apart from the request concurrency budget: a +// tail holds its slot for minutes, and counting it there would starve the +// same user's dashboard panels. +type limiter struct { + cfg LimiterConfig + now func() time.Time + mu sync.Mutex + active map[string]int + tails map[string]int + total int + buckets map[string]*bucket +} + +type bucket struct { + tokens float64 + last time.Time +} + +func newLimiter(cfg LimiterConfig) *limiter { + return &limiter{ + cfg: cfg, + now: time.Now, + active: map[string]int{}, + tails: map[string]int{}, + buckets: map[string]*bucket{}, + } +} + +// allowRate takes a token from the user's bucket; when it is empty, it +// returns how long until the next token. +func (l *limiter) allowRate(user string) (time.Duration, bool) { + if l.cfg.RatePerUser <= 0 { + return 0, true + } + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + burst := float64(max(l.cfg.RateBurst, 1)) + b, ok := l.buckets[user] + if !ok { + if len(l.buckets) >= ttlCacheMaxEntries { + // Every bucket still refilling means that many users within one + // refill period; a newcomer waits for the first to drop out + // rather than grow the map past its bound. + if soonest := l.sweepBuckets(now, burst); len(l.buckets) >= ttlCacheMaxEntries { + return soonest, false + } + } + b = &bucket{tokens: burst, last: now} + l.buckets[user] = b + } + b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*l.cfg.RatePerUser) + b.last = now + if b.tokens < 1 { + wait, ok := scaleDuration((1-b.tokens)/l.cfg.RatePerUser, time.Second) + if !ok { + wait = math.MaxInt64 + } + return wait, false + } + b.tokens-- + return 0, true +} + +// sweepBuckets drops buckets that have refilled completely; recreating one +// later starts it full, which is the same state. It returns how long until +// the first remaining bucket is full. +func (l *limiter) sweepBuckets(now time.Time, burst float64) time.Duration { + soonest := time.Duration(math.MaxInt64) + for user, b := range l.buckets { + missing := burst - (b.tokens + now.Sub(b.last).Seconds()*l.cfg.RatePerUser) + if missing <= 0 { + delete(l.buckets, user) + continue + } + if wait, ok := scaleDuration(missing/l.cfg.RatePerUser, time.Second); ok { + soonest = min(soonest, max(wait, time.Nanosecond)) + } + } + return soonest +} + +func (l *limiter) acquire(user string) (func(), bool) { + l.mu.Lock() + defer l.mu.Unlock() + if l.cfg.MaxConcurrentPerUser > 0 && l.active[user] >= l.cfg.MaxConcurrentPerUser { + return nil, false + } + l.active[user]++ + return l.releaser(func() { decrement(l.active, user) }), true +} + +func (l *limiter) acquireTail(user string) (func(), bool) { + l.mu.Lock() + defer l.mu.Unlock() + if l.cfg.MaxTailsPerUser > 0 && l.tails[user] >= l.cfg.MaxTailsPerUser { + return nil, false + } + if l.cfg.MaxTails > 0 && l.total >= l.cfg.MaxTails { + return nil, false + } + l.tails[user]++ + l.total++ + return l.releaser(func() { + decrement(l.tails, user) + l.total-- + }), true +} + +func (l *limiter) openTails() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.total +} + +func (l *limiter) releaser(release func()) func() { + var once sync.Once + return func() { + once.Do(func() { + l.mu.Lock() + defer l.mu.Unlock() + release() + }) + } +} + +func decrement(m map[string]int, user string) { + if m[user] <= 1 { + delete(m, user) + return + } + m[user]-- +} diff --git a/internal/observabilityproxy/limits_test.go b/internal/observabilityproxy/limits_test.go new file mode 100644 index 000000000..d7f4c76f4 --- /dev/null +++ b/internal/observabilityproxy/limits_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "testing" + "time" +) + +func TestLimiterConcurrency(t *testing.T) { + l := newLimiter(LimiterConfig{MaxConcurrentPerUser: 2}) + r1, ok1 := l.acquire("alice") + _, ok2 := l.acquire("alice") + if !ok1 || !ok2 { + t.Fatal("first two requests refused") + } + if _, ok := l.acquire("alice"); ok { + t.Fatal("third concurrent request admitted") + } + if _, ok := l.acquire("bob"); !ok { + t.Fatal("another user is limited by alice's requests") + } + r1() + r1() + if _, ok := l.acquire("alice"); !ok { + t.Fatal("slot not returned on release") + } + if _, ok := l.acquire("alice"); ok { + t.Fatal("double release freed a second slot") + } +} + +func TestLimiterTails(t *testing.T) { + l := newLimiter(LimiterConfig{MaxConcurrentPerUser: 1, MaxTailsPerUser: 2, MaxTails: 3}) + if _, ok := l.acquire("alice"); !ok { + t.Fatal("request refused") + } + a1, ok1 := l.acquireTail("alice") + _, ok2 := l.acquireTail("alice") + if !ok1 || !ok2 { + t.Fatal("tails share the request concurrency budget") + } + if _, ok := l.acquireTail("alice"); ok { + t.Fatal("per-user tail cap not enforced") + } + if _, ok := l.acquireTail("bob"); !ok { + t.Fatal("bob refused below the total cap") + } + if _, ok := l.acquireTail("carol"); ok { + t.Fatal("total tail cap not enforced") + } + if got := l.openTails(); got != 3 { + t.Fatalf("openTails = %d, want 3", got) + } + a1() + if _, ok := l.acquireTail("carol"); !ok { + t.Fatal("total slot not returned on release") + } +} + +func TestLimiterRate(t *testing.T) { + now := time.Unix(1000, 0) + l := newLimiter(LimiterConfig{RatePerUser: 2, RateBurst: 2}) + l.now = func() time.Time { return now } + for i := range 2 { + if !allowed(l, "alice") { + t.Fatalf("burst request %d refused", i+1) + } + } + if wait, ok := l.allowRate("alice"); ok || wait != 500*time.Millisecond { + t.Fatalf("request over the burst: admitted %v, wait %s, want refused with 500ms", ok, wait) + } + if !allowed(l, "bob") { + t.Fatal("bob limited by alice's rate") + } + now = now.Add(500 * time.Millisecond) + if !allowed(l, "alice") { + t.Fatal("token not refilled after 1/rate") + } + now = now.Add(100 * time.Millisecond) + if wait, ok := l.allowRate("alice"); ok || wait != 400*time.Millisecond { + t.Fatalf("partly refilled bucket: admitted %v, wait %s, want refused with 400ms", ok, wait) + } +} + +func TestLimiterRateDisabled(t *testing.T) { + l := newLimiter(LimiterConfig{}) + for range 100 { + if !allowed(l, "alice") { + t.Fatal("rate limit applied while disabled") + } + } +} + +func allowed(l *limiter, user string) bool { + _, ok := l.allowRate(user) + return ok +} + +func TestLimiterRateWaitAtTinyRate(t *testing.T) { + l := newLimiter(LimiterConfig{RatePerUser: 1e-12, RateBurst: 1}) + allowed(l, "alice") + if wait, ok := l.allowRate("alice"); ok || wait <= 0 { + t.Fatalf("empty bucket at a tiny rate: admitted %v, wait %s, want refused with a positive wait", ok, wait) + } +} + +// TestLimiterRateBucketsBounded fills the bucket map with users that are +// all still refilling: a new user must be refused rather than grow it, and +// admitted once a bucket can be dropped. +func TestLimiterRateBucketsBounded(t *testing.T) { + now := time.Unix(1000, 0) + l := newLimiter(LimiterConfig{RatePerUser: 2, RateBurst: 2}) + l.now = func() time.Time { return now } + for i := range ttlCacheMaxEntries { + if !allowed(l, fmt.Sprintf("user-%d", i)) { + t.Fatalf("user-%d refused", i) + } + now = now.Add(time.Microsecond) + } + wait, ok := l.allowRate("newcomer") + if ok || wait <= 0 { + t.Fatalf("newcomer on a full map: admitted %v, wait %s; want refused with a positive wait", ok, wait) + } + if n := len(l.buckets); n > ttlCacheMaxEntries { + t.Fatalf("bucket map grew to %d, above %d", n, ttlCacheMaxEntries) + } + now = now.Add(wait) + if !allowed(l, "newcomer") { + t.Fatalf("newcomer still refused after the advertised wait %s", wait) + } +} diff --git a/internal/observabilityproxy/logsparams.go b/internal/observabilityproxy/logsparams.go new file mode 100644 index 000000000..ec6043c6f --- /dev/null +++ b/internal/observabilityproxy/logsparams.go @@ -0,0 +1,426 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "math" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +// LogsLimits bound what one logs request may make vlselect do. +type LogsLimits struct { + DefaultRange time.Duration + MaxRange time.Duration + MaxBuckets int + DefaultLimit int + MaxLimit int + MaxTailResume time.Duration +} + +// tailOffset is vlselect's default tail offset. The proxy never forwards a +// client offset, so the resume arithmetic can rely on this value. +const tailOffset = 5 * time.Second + +const maxHitsFields = 10 + +var logsEndpoints = map[string]bool{ + "query": true, "hits": true, "stats_query_range": true, "tail": true, + "field_values": true, "stream_field_values": true, "facets": true, +} + +// BuildLogsParams rebuilds the upstream parameters from scratch: only the +// allowlisted parameters of the endpoint are read from the client, and the +// namespace scope is added as the only extra_stream_filters value. +// VictoriaLogs ANDs extra_stream_filters into the query and every subquery +// (union, join, in()), which is what lets the query itself pass through +// unparsed. +func BuildLogsParams(endpoint string, in url.Values, namespace string, now time.Time, lim LogsLimits) (url.Values, *HTTPError) { + if !logsEndpoints[endpoint] { + return nil, errorf(http.StatusNotFound, "not found") + } + query := in.Get("query") + if query == "" { + query = "*" + } + // ignore_global_time_filter lifts the time window this function + // enforces, and on tail rescans the whole retention every tick. A + // substring match for it is bypassable (VictoriaLogs unquotes a quoted + // option name and joins words split by a comment), and no option is + // needed by a tenant, so any options(...) block is refused. + if hasOptionsBlock(query) { + return nil, errorf(http.StatusBadRequest, "query options are not allowed") + } + out := url.Values{} + out.Set("query", query) + out.Set("extra_stream_filters", "{kubernetes_namespace_name="+strconv.Quote(namespace)+"}") + + if endpoint == "tail" { + return out, tailParams(in, out, now, lim) + } + + start, end, herr := window(in, now, lim) + if herr != nil { + return nil, herr + } + out.Set("start", start.Format(time.RFC3339Nano)) + out.Set("end", end.Format(time.RFC3339Nano)) + + switch endpoint { + case "query": + herr = setLimit(out, in, "limit", lim.DefaultLimit, lim.MaxLimit) + case "hits": + if herr = setStep(out, in, start, end, lim); herr != nil { + return nil, herr + } + fields := in["field"] + if len(fields) > maxHitsFields { + return nil, errorf(http.StatusBadRequest, "at most %d field parameters are allowed", maxHitsFields) + } + for _, f := range fields { + if f != "" { + out.Add("field", f) + } + } + // vlselect returns every group without fields_limit or with 0, so + // the cap is sent either way. + herr = setLimit(out, in, "fields_limit", lim.MaxLimit, lim.MaxLimit) + case "stats_query_range": + herr = setStep(out, in, start, end, lim) + case "field_values", "stream_field_values": + if in.Get("field") == "" { + return nil, errorf(http.StatusBadRequest, "field is required") + } + out.Set("field", in.Get("field")) + if f := in.Get("filter"); f != "" { + out.Set("filter", f) + } + herr = setLimit(out, in, "limit", lim.DefaultLimit, lim.MaxLimit) + case "facets": + for _, k := range []string{"limit", "max_values_per_field", "max_value_len"} { + if k == "limit" || in.Has(k) { + if herr = setLimit(out, in, k, lim.DefaultLimit, lim.MaxLimit); herr != nil { + return nil, herr + } + } + } + if v := in.Get("keep_const_fields"); v == "1" || v == "true" { + out.Set("keep_const_fields", "1") + } + } + if herr != nil { + return nil, herr + } + return out, nil +} + +// window derives a finite [start, end]: VictoriaLogs otherwise starts at +// the oldest stored entry and scans the whole retention. +func window(in url.Values, now time.Time, lim LogsLimits) (time.Time, time.Time, *HTTPError) { + end := now + if v := in.Get("end"); v != "" { + t, err := parseTime(v, now) + if err != nil { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "invalid end: %v", err) + } + end = t + } + start := end.Add(-lim.DefaultRange) + if v := in.Get("start"); v != "" { + t, err := parseTime(v, now) + if err != nil { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "invalid start: %v", err) + } + start = t + } + if start.After(end) { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end") + } + if end.Sub(start) > lim.MaxRange { + start = end.Add(-lim.MaxRange) + } + if herr := sendableTime("end", end); herr != nil { + return time.Time{}, time.Time{}, herr + } + if herr := sendableTime("start", start); herr != nil { + return time.Time{}, time.Time{}, herr + } + return start.UTC(), end.UTC(), nil +} + +// setStep raises step so that hits and stats_query_range, which return one +// bucket per step, return at most MaxBuckets buckets, and caps it so the +// window cannot grow through it: vlselect widens [start, end] outward to +// step boundaries (alignStartEndToStep), by up to one step on each side, +// so a 100y step would otherwise scan the whole retention. The window is +// shrunk from the start until the widened span fits MaxRange. +func setStep(out, in url.Values, start, end time.Time, lim LogsLimits) *HTTPError { + span := end.Sub(start) + // vlselect returns a zero-filled bucket for every step boundary from + // the aligned start through the aligned end, one more than span/step + // whenever start is off a boundary, so one bucket is held back. + spans := time.Duration(lim.MaxBuckets - 1) + minStep := max((span+spans-1)/spans, time.Second) + step := minStep + if v := in.Get("step"); v != "" { + d, err := parseDuration(v) + if err != nil || d <= 0 { + return errorf(http.StatusBadRequest, "invalid step %q", v) + } + step = max(min(d, span, lim.MaxRange/3), minStep) + } + // vlselect gets whole milliseconds; rounding down would add a bucket. + if r := step % time.Millisecond; r != 0 { + step += time.Millisecond - r + } + if span+2*step > lim.MaxRange { + start = end.Add(-max(lim.MaxRange-2*step, 0)) + } + out.Set("step", formatDuration(step)) + out.Set("start", start.Format(time.RFC3339Nano)) + return nil +} + +// setLimit refuses an empty value rather than read it as the default: a +// cap the client sent but left blank is malformed, the same for every cap, +// where reading it as the default would hide a client bug. +func setLimit(out, in url.Values, key string, def, maxv int) *HTTPError { + n := def + if in.Has(key) && in.Get(key) == "" { + return errorf(http.StatusBadRequest, "invalid %s \"\"", key) + } + if v := in.Get(key); v != "" { + parsed, err := strconv.Atoi(v) + if err != nil || parsed <= 0 { + return errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + n = parsed + } + out.Set(key, strconv.Itoa(min(n, maxv))) + return nil +} + +// tailParams turns a resume point into a capped start_offset. A reconnect +// older than the cap resumes at the cap; the gap needs a historical query. +func tailParams(in, out url.Values, now time.Time, lim LogsLimits) *HTTPError { + var start, end time.Time + for _, k := range []string{"start", "end"} { + v := in.Get(k) + if v == "" { + continue + } + t, err := parseTime(v, now) + if err != nil { + return errorf(http.StatusBadRequest, "invalid %s: %v", k, err) + } + if herr := sendableTime(k, t); herr != nil { + return herr + } + out.Set(k, t.UTC().Format(time.RFC3339Nano)) + if k == "start" { + start = t + } else { + end = t + } + } + if !start.IsZero() && !end.IsZero() && start.After(end) { + return errorf(http.StatusBadRequest, "start is after end") + } + + since, offset := in.Get("since"), in.Get("start_offset") + var d time.Duration + switch { + case since != "" && offset != "": + return errorf(http.StatusBadRequest, "since and start_offset are mutually exclusive") + case since != "": + t, err := parseTime(since, now) + if err != nil { + return errorf(http.StatusBadRequest, "invalid since: %v", err) + } + d = now.Add(-tailOffset).Sub(t) + case offset != "": + parsed, err := parseDuration(offset) + if err != nil { + return errorf(http.StatusBadRequest, "invalid start_offset %q", offset) + } + d = parsed + default: + return nil + } + out.Set("start_offset", formatDuration(min(max(d, 0), lim.MaxTailResume))) + return nil +} + +// formatDuration writes whole milliseconds, a form every VictoriaLogs +// duration parser accepts. +func formatDuration(d time.Duration) string { + return fmt.Sprintf("%dms", d.Milliseconds()) +} + +var durationToken = regexp.MustCompile(`^(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) + +var durationUnits = map[string]time.Duration{ + "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, + "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, +} + +// parseDuration accepts the LogsQL duration forms (1h30m, 1d, 2w) and bare +// seconds; Go's parser knows neither d nor w. +func parseDuration(s string) (time.Duration, error) { + // Bare seconds only as digits with an optional fraction: ParseFloat + // would also take signs, exponents, Inf and NaN. + if whole, frac, ok := strings.Cut(s, "."); isDigits(whole) && (!ok || isDigits(frac)) { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("invalid duration %q", s) + } + d, ok := scaleDuration(f, time.Second) + if !ok { + return 0, fmt.Errorf("duration %q out of range", s) + } + return d, nil + } + if s == "" { + return 0, fmt.Errorf("empty duration") + } + var total time.Duration + for rest := s; rest != ""; { + m := durationToken.FindStringSubmatch(rest) + if m == nil { + return 0, fmt.Errorf("invalid duration %q", s) + } + f, _ := strconv.ParseFloat(m[1], 64) + d, ok := scaleDuration(f, durationUnits[m[2]]) + if !ok || total > math.MaxInt64-d { + return 0, fmt.Errorf("duration %q out of range", s) + } + total += d + rest = rest[len(m[0]):] + } + return total, nil +} + +// scaleDuration refuses what does not fit a Duration: Go leaves an +// out-of-range float to int conversion implementation-defined. +func scaleDuration(f float64, unit time.Duration) (time.Duration, bool) { + v := f * float64(unit) + // float64(math.MaxInt64) rounds up to 2^63, which is already too large. + if v >= float64(math.MaxInt64) { + return 0, false + } + return time.Duration(v), true +} + +// sendableTime refuses a time vlselect cannot hold: it reads the RFC3339 +// the proxy sends through time.UnixNano, which is undefined outside the +// int64 range (1677..2262) and would move the window anywhere. Only times +// actually sent are checked; an older start is clamped first. +func sendableTime(key string, t time.Time) *HTTPError { + if t.Before(time.Unix(0, math.MinInt64)) || t.After(time.Unix(0, math.MaxInt64)) { + return errorf(http.StatusBadRequest, "invalid %s: %s is outside 1677..2262", key, t.UTC().Format(time.RFC3339Nano)) + } + return nil +} + +// parseTime accepts "now", unix seconds with an optional fraction, RFC3339, +// and a duration meaning that long before now (VictoriaLogs' "start=3h"). +// Anything else is refused rather than guessed, since a misread timestamp +// silently moves the window. +func parseTime(s string, now time.Time) (time.Time, error) { + if s == "now" { + return now, nil + } + if sec, frac, ok := strings.Cut(s, "."); isDigits(sec) && (!ok || isDigits(frac)) { + // vlselect reads the unit from the digit count (four digits are a + // year, 13 are milliseconds); only unambiguous seconds are taken. + if len(sec) < 5 || len(sec) > 10 { + return time.Time{}, fmt.Errorf("numeric time %q is not unix seconds", s) + } + secs, err := strconv.ParseInt(sec, 10, 64) + if err != nil { + return time.Time{}, err + } + var nanos int64 + if ok { + frac = (frac + "000000000")[:9] + nanos, _ = strconv.ParseInt(frac, 10, 64) + } + return time.Unix(secs, nanos), nil + } + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + if d, err := parseDuration(s); err == nil { + return now.Add(-d), nil + } + return time.Time{}, fmt.Errorf("unsupported time %q", s) +} + +// hasOptionsBlock reports whether the options keyword is followed by "(", +// skipping whitespace and "#" comments the way the LogsQL lexer does. A +// quoted "options" is never the keyword, but matching it too only refuses +// a search for that literal phrase. +func hasOptionsBlock(query string) bool { + lower := strings.ToLower(query) + for i := strings.Index(lower, "options"); i >= 0; { + rest := lower[i+len("options"):] + for rest != "" { + r, size := utf8.DecodeRuneInString(rest) + switch { + case unicode.IsSpace(r): + rest = rest[size:] + continue + case r == '#': + if n := strings.IndexByte(rest, '\n'); n >= 0 { + rest = rest[n+1:] + } else { + rest = "" + } + continue + } + break + } + if strings.HasPrefix(rest, "(") { + return true + } + next := strings.Index(lower[i+1:], "options") + if next < 0 { + return false + } + i += next + 1 + } + return false +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/internal/observabilityproxy/logsparams_test.go b/internal/observabilityproxy/logsparams_test.go new file mode 100644 index 000000000..82ed29190 --- /dev/null +++ b/internal/observabilityproxy/logsparams_test.go @@ -0,0 +1,434 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +var testNow = time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + +func testLimits() LogsLimits { + return LogsLimits{ + DefaultRange: time.Hour, + MaxRange: 7 * 24 * time.Hour, + MaxBuckets: 1000, + DefaultLimit: 1000, + MaxLimit: 5000, + MaxTailResume: 15 * time.Minute, + } +} + +const wantScope = `{kubernetes_namespace_name="tenant-a"}` + +func build(t *testing.T, endpoint string, in url.Values) url.Values { + t.Helper() + out, err := BuildLogsParams(endpoint, in, "tenant-a", testNow, testLimits()) + if err != nil { + t.Fatalf("BuildLogsParams(%s, %v) error: %v", endpoint, in, err) + } + return out +} + +func buildErr(t *testing.T, endpoint string, in url.Values) *HTTPError { + t.Helper() + out, err := BuildLogsParams(endpoint, in, "tenant-a", testNow, testLimits()) + if err == nil { + t.Fatalf("BuildLogsParams(%s, %v) = %v, want error", endpoint, in, out) + } + return err +} + +func TestBuildLogsParamsAllowlist(t *testing.T) { + allowed := map[string][]string{ + "query": {"query", "start", "end", "limit"}, + "hits": {"query", "start", "end", "step", "field", "fields_limit"}, + "stats_query_range": {"query", "start", "end", "step"}, + "field_values": {"query", "start", "end", "field", "limit", "filter"}, + "stream_field_values": {"query", "start", "end", "field", "limit", "filter"}, + "facets": {"query", "start", "end", "limit", "max_values_per_field", "max_value_len", "keep_const_fields"}, + "tail": {"query", "start", "end", "start_offset"}, + } + hostile := url.Values{ + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_filters[]": {`kubernetes_namespace_name:~".*"`}, + "extra_stream_filters": {`{kubernetes_namespace_name=~".*"}`}, + "AccountID": {"7"}, + "ProjectID": {"7"}, + "tenant": {"tenant-b"}, + "offset": {"1h"}, + "refresh_interval": {"1ms"}, + "ignore_pipes": {"1"}, + "format": {"csv"}, + "timeout": {"1h"}, + "query": {"error"}, + "field": {"kubernetes_pod_name"}, + "filter": {"api"}, + "keep_const_fields": {"1"}, + "since": {"2026-11-04T11:59:00Z"}, + } + for endpoint, keys := range allowed { + t.Run(endpoint, func(t *testing.T) { + out := build(t, endpoint, hostile) + want := map[string]bool{"extra_stream_filters": true} + for _, k := range keys { + want[k] = true + } + for k := range out { + if !want[k] { + t.Errorf("forwarded non-allowlisted parameter %q", k) + } + } + if got := out["extra_stream_filters"]; len(got) != 1 || got[0] != wantScope { + t.Errorf("extra_stream_filters = %q, want exactly [%s]", got, wantScope) + } + if _, ok := out["extra_filters"]; ok { + t.Error("extra_filters forwarded") + } + }) + } +} + +func TestBuildLogsParamsRejects(t *testing.T) { + tests := []struct { + name string + endpoint string + in url.Values + code int + }{ + {"unknown endpoint", "tenant_ids", url.Values{}, http.StatusNotFound}, + // Malformed rather than a request for the default. + {"empty fields_limit", "hits", url.Values{"field": {"level"}, "fields_limit": {""}}, http.StatusBadRequest}, + {"empty limit", "query", url.Values{"limit": {""}}, http.StatusBadRequest}, + {"empty facets cap", "facets", url.Values{"max_values_per_field": {""}}, http.StatusBadRequest}, + {"stats_query is not proxied", "stats_query", url.Values{}, http.StatusNotFound}, + {"ignore_global_time_filter", "query", url.Values{"query": {`* options(ignore_global_time_filter=true)`}}, http.StatusBadRequest}, + {"ignore_global_time_filter in subquery", "hits", url.Values{"query": {`* | union (options(ignore_global_time_filter=true) error)`}}, http.StatusBadRequest}, + {"ignore_global_time_filter on tail", "tail", url.Values{"query": {`options(IGNORE_GLOBAL_TIME_FILTER=true) *`}}, http.StatusBadRequest}, + {"options with a quoted escaped name", "query", url.Values{"query": {`options("ignore_global_time_filte\x72"=true) *`}}, http.StatusBadRequest}, + {"options with a comment inside the name", "query", url.Values{"query": {"options(ignore_global#c\n_time_filter=true) *"}}, http.StatusBadRequest}, + {"options after a comment and unicode space", "tail", url.Values{"query": {"OPTIONS #c\n\u00a0(concurrency=1000) *"}}, http.StatusBadRequest}, + {"inverted range", "query", url.Values{"start": {"2026-11-04T11:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, http.StatusBadRequest}, + {"inverted range on tail", "tail", url.Values{"start": {"2026-11-04T11:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, http.StatusBadRequest}, + {"bad start", "query", url.Values{"start": {"yesterday"}}, http.StatusBadRequest}, + {"millisecond timestamp", "query", url.Values{"start": {"1793786400000"}}, http.StatusBadRequest}, + {"year-like number", "query", url.Values{"start": {"2026"}}, http.StatusBadRequest}, + {"exponent time", "query", url.Values{"start": {"1.7937864e9"}}, http.StatusBadRequest}, + {"signed time", "query", url.Values{"start": {"+1793786400"}}, http.StatusBadRequest}, + {"exponent duration", "query", url.Values{"start": {"3e2"}}, http.StatusBadRequest}, + {"trailing dot", "query", url.Values{"start": {"300."}}, http.StatusBadRequest}, + {"infinite time", "query", url.Values{"end": {"Inf"}}, http.StatusBadRequest}, + // vlselect keeps times as int64 nanoseconds (time.UnixNano), which + // is undefined outside 1677..2262; a time sent there must fit. + {"end before int64 nanoseconds", "query", url.Values{"start": {"0001-01-01T00:00:00Z"}, "end": {"0001-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"end past int64 nanoseconds", "query", url.Values{"end": {"9999-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"unix seconds past int64 nanoseconds", "query", url.Values{"start": {"9999999990"}, "end": {"9999999999"}}, http.StatusBadRequest}, + {"tail end past int64 nanoseconds", "tail", url.Values{"end": {"9999-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"NaN time", "query", url.Values{"start": {"NaN"}}, http.StatusBadRequest}, + {"NaN step", "hits", url.Values{"step": {"NaN"}}, http.StatusBadRequest}, + {"NaN start_offset", "tail", url.Values{"start_offset": {"NaN"}}, http.StatusBadRequest}, + {"bad limit", "query", url.Values{"limit": {"-1"}}, http.StatusBadRequest}, + {"zero limit", "query", url.Values{"limit": {"0"}}, http.StatusBadRequest}, + {"bad step", "hits", url.Values{"step": {"0s"}}, http.StatusBadRequest}, + {"field required", "field_values", url.Values{}, http.StatusBadRequest}, + {"too many hits fields", "hits", url.Values{"field": strings.Fields("a b c d e f g h i j k")}, http.StatusBadRequest}, + {"since and start_offset", "tail", url.Values{"since": {"2026-11-04T11:59:00Z"}, "start_offset": {"1m"}}, http.StatusBadRequest}, + {"bad since", "tail", url.Values{"since": {"soon"}}, http.StatusBadRequest}, + {"start_offset summing past int64", "tail", url.Values{"start_offset": {"99999999999h99999999999h1s"}}, http.StatusBadRequest}, + {"step summing past int64", "hits", url.Values{"step": {"2562047h2562047h"}}, http.StatusBadRequest}, + {"step token past int64", "hits", url.Values{"step": {"99999999999h"}}, http.StatusBadRequest}, + {"bare-seconds step past int64", "hits", url.Values{"step": {"99999999999999999999"}}, http.StatusBadRequest}, + {"relative start summing past int64", "query", url.Values{"start": {"106751d1d"}}, http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := buildErr(t, tt.endpoint, tt.in); err.Code != tt.code { + t.Fatalf("code = %d (%s), want %d", err.Code, err.Message, tt.code) + } + }) + } +} + +// VictoriaLogs propagates extra_stream_filters into every subquery +// (docs.victoriametrics.com/victorialogs/querying/), so union and in() +// queries are forwarded verbatim with the scope beside them. +func TestBuildLogsParamsSubqueriesKeepScope(t *testing.T) { + for _, q := range []string{ + `* | union ({kubernetes_namespace_name="tenant-b"})`, + `_stream_id:in({kubernetes_namespace_name="tenant-b"} | fields _stream_id)`, + } { + out := build(t, "query", url.Values{"query": {q}}) + if out.Get("query") != q || out.Get("extra_stream_filters") != wantScope { + t.Errorf("query %q forwarded as %v", q, out) + } + } +} + +func TestBuildLogsParamsWindow(t *testing.T) { + ts := func(s string) string { + v, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) + } + return v.UTC().Format(time.RFC3339Nano) + } + tests := []struct { + name string + in url.Values + wantStart, wantEnd string + }{ + {"defaults", url.Values{}, ts("2026-11-04T11:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"start only", url.Values{"start": {"2026-11-04T09:00:00Z"}}, ts("2026-11-04T09:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"end only", url.Values{"end": {"2026-11-03T00:00:00Z"}}, ts("2026-11-02T23:00:00Z"), ts("2026-11-03T00:00:00Z")}, + {"unix seconds", url.Values{"start": {"1793786400"}, "end": {"1793788200.5"}}, "2026-11-04T10:00:00Z", "2026-11-04T10:30:00.5Z"}, + {"rfc3339 with offset", url.Values{"start": {"2026-11-04T13:00:00+03:00"}}, ts("2026-11-04T10:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"relative", url.Values{"start": {"3h"}, "end": {"now"}}, ts("2026-11-04T09:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"relative days", url.Values{"start": {"1d"}}, ts("2026-11-03T12:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"over-wide clamped", url.Values{"start": {"2026-01-01T00:00:00Z"}}, ts("2026-10-28T12:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"empty range allowed", url.Values{"start": {"2026-11-04T10:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, ts("2026-11-04T10:00:00Z"), ts("2026-11-04T10:00:00Z")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, endpoint := range []string{"query", "field_values", "facets"} { + in := url.Values{"field": {"f"}} + for k, v := range tt.in { + in[k] = v + } + out := build(t, endpoint, in) + if out.Get("start") != tt.wantStart || out.Get("end") != tt.wantEnd { + t.Errorf("%s: window = [%s, %s], want [%s, %s]", endpoint, out.Get("start"), out.Get("end"), tt.wantStart, tt.wantEnd) + } + } + }) + } +} + +func TestBuildLogsParamsStepAndLimit(t *testing.T) { + day := url.Values{"start": {"2026-11-03T12:00:00Z"}} + with := func(base url.Values, k, v string) url.Values { + c := url.Values{} + for key, vals := range base { + c[key] = vals + } + c.Set(k, v) + return c + } + // 24h over 999 steps, 1000 buckets once aligned, is 86486.5ms, rounded + // up to whole milliseconds. + if got := build(t, "hits", with(day, "step", "1s")).Get("step"); got != "86487ms" { + t.Errorf("tiny step = %s, want clamped to 86487ms", got) + } + if got := build(t, "stats_query_range", with(day, "step", "1h")).Get("step"); got != "3600000ms" { + t.Errorf("1h step = %s", got) + } + if got := build(t, "stats_query_range", with(day, "step", "1w")).Get("step"); got != "86400000ms" { + t.Errorf("1w step on a day window = %s, want capped at the window", got) + } + if got := build(t, "hits", with(day, "step", "3600")).Get("step"); got != "3600000ms" { + t.Errorf("bare-seconds step = %s, want 3600000ms", got) + } + if got := build(t, "hits", with(day, "step", "5400.5")).Get("step"); got != "5400500ms" { + t.Errorf("fractional bare-seconds step = %s, want 5400500ms", got) + } + if got := build(t, "hits", day).Get("step"); got != "86487ms" { + t.Errorf("omitted step = %s, want the bucket-bounded minimum", got) + } + if got := build(t, "hits", url.Values{"start": {"2026-11-04T11:59:59Z"}}).Get("step"); got != "1000ms" { + t.Errorf("omitted step on a tiny window = %s, want 1000ms floor", got) + } + + if got := build(t, "query", url.Values{}).Get("limit"); got != "1000" { + t.Errorf("default limit = %s", got) + } + if got := build(t, "query", url.Values{"limit": {"10"}}).Get("limit"); got != "10" { + t.Errorf("limit = %s", got) + } + for _, endpoint := range []string{"query", "field_values", "facets"} { + in := url.Values{"limit": {"1000000"}, "field": {"f"}} + if got := build(t, endpoint, in).Get("limit"); got != "5000" { + t.Errorf("%s: limit = %s, want clamped to 5000", endpoint, got) + } + } + facets := build(t, "facets", url.Values{"max_values_per_field": {"99999"}, "max_value_len": {"99999"}}) + if facets.Get("max_values_per_field") != "5000" || facets.Get("max_value_len") != "5000" { + t.Errorf("facets caps not clamped: %v", facets) + } + // Without fields_limit vlselect returns every group, so the cap is + // sent even when the client sends none. + if got := build(t, "hits", url.Values{"field": {"level"}}).Get("fields_limit"); got != "5000" { + t.Errorf("hits without fields_limit forwards %q, want the 5000 cap", got) + } + hits := build(t, "hits", url.Values{"field": {"b", "a"}, "fields_limit": {"99999"}}) + fields := hits["field"] + sort.Strings(fields) + if strings.Join(fields, ",") != "a,b" || hits.Get("fields_limit") != "5000" { + t.Errorf("hits fields = %v", hits) + } +} + +func TestBuildLogsParamsTail(t *testing.T) { + tests := []struct { + name string + in url.Values + want string + }{ + {"live only", url.Values{}, ""}, + // now - 5s tail offset - since = 55s + {"since within cap", url.Values{"since": {"2026-11-04T11:59:00Z"}}, "55000ms"}, + {"since older than cap", url.Values{"since": {"2026-11-04T08:00:00Z"}}, "900000ms"}, + {"since in the future", url.Values{"since": {"2026-11-04T13:00:00Z"}}, "0ms"}, + {"start_offset within cap", url.Values{"start_offset": {"2m"}}, "120000ms"}, + {"start_offset over cap", url.Values{"start_offset": {"30d"}}, "900000ms"}, + {"start_offset in bare seconds", url.Values{"start_offset": {"120"}}, "120000ms"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := build(t, "tail", tt.in) + if got := out.Get("start_offset"); got != tt.want { + t.Errorf("start_offset = %q, want %q", got, tt.want) + } + if out.Has("start") || out.Has("end") || out.Has("limit") { + t.Errorf("tail got a derived window or limit: %v", out) + } + }) + } + out := build(t, "tail", url.Values{"start": {"2026-11-04T11:00:00Z"}}) + if out.Get("start") != "2026-11-04T11:00:00Z" { + t.Errorf("tail start = %q, want the client value ANDed into each tick", out.Get("start")) + } +} + +// alignedWindow repeats vlselect's alignStartEndToStep (offset 0): with a +// step it widens [start, end] to step boundaries, end always by at least +// one more step. +func alignedWindow(t *testing.T, out url.Values) time.Duration { + t.Helper() + parse := func(k string) int64 { + v, err := time.Parse(time.RFC3339Nano, out.Get(k)) + if err != nil { + t.Fatalf("%s: %v", k, err) + } + return v.UnixNano() + } + step, err := parseDuration(out.Get("step")) + if err != nil { + t.Fatal(err) + } + start, end, st := parse("start"), parse("end"), int64(step) + start -= start % st + end += st - end%st + return time.Duration(end - start) +} + +func TestBuildLogsParamsStepCannotWidenWindow(t *testing.T) { + lim := testLimits() + for _, endpoint := range []string{"hits", "stats_query_range"} { + for _, in := range []url.Values{ + {"step": {"100y"}}, + {"step": {"100y"}, "start": {"2026-01-01T00:00:00Z"}}, + {"step": {"3d"}, "start": {"7d"}}, + {"start": {"7d"}}, + {"step": {"1s"}, "start": {"7d"}}, + } { + out := build(t, endpoint, in) + if got := alignedWindow(t, out); got > lim.MaxRange { + t.Errorf("%s %v: vlselect would scan %s, more than %s", endpoint, in, got, lim.MaxRange) + } + } + } +} + +// TestBuildLogsParamsStepKeepsBucketCap counts buckets the way vlselect +// returns them: one per step boundary from the aligned start through the +// aligned end, zero-filled, which is one more than the unaligned span +// holds whenever start is not on a boundary. +func TestBuildLogsParamsStepKeepsBucketCap(t *testing.T) { + lim := testLimits() + // Spans that MaxBuckets does not divide into whole milliseconds, and + // unaligned starts of spans it does divide. + for _, start := range []string{ + "2026-11-03T12:00:00.0007Z", "2026-11-04T10:00:00.0013Z", "2026-11-04T11:00:00.0005Z", + "2026-11-01T12:00:00.5Z", "2026-11-04T11:00:00.25Z", + } { + out := build(t, "hits", url.Values{"start": {start}}) + step, err := parseDuration(out.Get("step")) + if err != nil { + t.Fatal(err) + } + if buckets := alignedWindow(t, out) / step; int(buckets) > lim.MaxBuckets { + t.Errorf("start %s: step %s gives %d buckets, more than %d", start, out.Get("step"), buckets, lim.MaxBuckets) + } + } +} + +func TestParseDurationBounds(t *testing.T) { + for _, in := range []string{"2562047h", "106751d", "9223372036s", "292y"} { + if d, err := parseDuration(in); err != nil || d <= 0 { + t.Errorf("parseDuration(%q) = %s, %v; want a positive duration", in, d, err) + } + } + for _, in := range []string{"2562048h", "106751d1d", "9223372037s", "9223372037", "293y", "1y292y"} { + if d, err := parseDuration(in); err == nil { + t.Errorf("parseDuration(%q) = %s; want an error past the int64 range", in, d) + } + } +} + +// TestBuildLogsParamsOldTimesClamped: only times that reach vlselect must +// fit its int64 nanoseconds. An old start is clamped to MaxRange before it +// is sent, and an old since becomes a start_offset capped at the resume +// limit, so both are served. +func TestBuildLogsParamsOldTimesClamped(t *testing.T) { + lim := testLimits() + out := build(t, "query", url.Values{"start": {"1900-01-01T00:00:00Z"}}) + if want := testNow.Add(-lim.MaxRange).Format(time.RFC3339Nano); out.Get("start") != want { + t.Errorf("start = %s, want clamped to %s", out.Get("start"), want) + } + out = build(t, "tail", url.Values{"since": {"1969-12-31T23:59:59Z"}}) + if got := out.Get("start_offset"); got != formatDuration(lim.MaxTailResume) { + t.Errorf("start_offset = %s, want the %s cap", got, lim.MaxTailResume) + } +} + +// TestBuildLogsParamsSendableBoundary pins the inclusive int64-nanosecond +// bound and that the refusal names the instant exactly. +func TestBuildLogsParamsSendableBoundary(t *testing.T) { + last := time.Unix(0, math.MaxInt64).UTC() + in := url.Values{"start": {last.Add(-time.Hour).Format(time.RFC3339Nano)}, "end": {last.Format(time.RFC3339Nano)}} + if _, herr := BuildLogsParams("query", in, "tenant-a", testNow, testLimits()); herr != nil { + t.Fatalf("the last int64 nanosecond refused: %v", herr) + } + past := last.Add(time.Nanosecond).Format(time.RFC3339Nano) + in = url.Values{"start": {last.Add(-time.Hour).Format(time.RFC3339Nano)}, "end": {past}} + _, herr := BuildLogsParams("query", in, "tenant-a", testNow, testLimits()) + if herr == nil || !strings.Contains(herr.Message, "invalid end") || !strings.Contains(herr.Message, past) { + t.Fatalf("one nanosecond past answered %v, want end refused naming %s", herr, past) + } + // Both out of range: the answer names end, deterministically. + for range 20 { + _, herr := BuildLogsParams("query", url.Values{"start": {"0001-01-01T00:00:00Z"}, "end": {"0001-01-02T00:00:00Z"}}, "tenant-a", testNow, testLimits()) + if herr == nil || !strings.Contains(herr.Message, "invalid end") { + t.Fatalf("answer = %v, want end named", herr) + } + } +} diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go new file mode 100644 index 000000000..56ee9e496 --- /dev/null +++ b/internal/observabilityproxy/route.go @@ -0,0 +1,148 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "net/http" + "strings" + + authorizationv1 "k8s.io/api/authorization/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +// RouteKind is one of the four Kubernetes-shaped roots the proxy serves. +type RouteKind int + +const ( + NamespaceLogs RouteKind = iota + 1 + NamespaceMetrics + ApplicationLog + ApplicationMetrics +) + +func (k RouteKind) String() string { + switch k { + case NamespaceLogs: + return "namespace-logs" + case NamespaceMetrics: + return "namespace-metrics" + case ApplicationLog: + return "application-log" + case ApplicationMetrics: + return "application-metrics" + } + return "unknown" +} + +const ( + coreGroup = "cozystack.io" + appsGroup = "apps.cozystack.io" +) + +// Route is a request path split into the scope it names and the upstream +// endpoint below the root. Plural and Name are set only on application roots. +type Route struct { + Kind RouteKind + Namespace string + Plural string + Name string + Endpoint string +} + +// HTTPError carries the status code the proxy answers with. +type HTTPError struct { + Code int + Message string +} + +func (e *HTTPError) Error() string { return fmt.Sprintf("%d: %s", e.Code, e.Message) } + +func errorf(code int, format string, args ...any) *HTTPError { + return &HTTPError{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// ParseRoute takes the escaped request path. The console nginx matches its +// location on the normalized path but forwards the original one, so a path +// that a normalizer would rewrite (dot segments, empty segments, any +// percent-encoding such as %2F or %2e) is refused instead of interpreted: +// the namespace the SubjectAccessReview names must be the one queried. +func ParseRoute(escapedPath string) (Route, *HTTPError) { + notFound := errorf(http.StatusNotFound, "not found") + if strings.Contains(escapedPath, "%") { + return Route{}, errorf(http.StatusBadRequest, "percent-encoded paths are not accepted") + } + segs := strings.Split(strings.TrimPrefix(escapedPath, "/"), "/") + if len(segs) > 0 && segs[len(segs)-1] == "" { + segs = segs[:len(segs)-1] + } + for _, s := range segs { + if s == "" || s == "." || s == ".." { + return Route{}, errorf(http.StatusBadRequest, "non-canonical path") + } + } + if len(segs) < 6 || segs[0] != "apis" || segs[2] != "v1alpha1" || segs[3] != "namespaces" { + return Route{}, notFound + } + + var r Route + var rest []string + switch { + case segs[1] == coreGroup && segs[5] == "logs": + r.Kind, rest = NamespaceLogs, segs[6:] + case segs[1] == coreGroup && segs[5] == "metrics": + r.Kind, rest = NamespaceMetrics, segs[6:] + case segs[1] == appsGroup && len(segs) >= 8 && segs[7] == "log": + r.Kind, rest = ApplicationLog, segs[8:] + case segs[1] == appsGroup && len(segs) >= 8 && segs[7] == "metrics": + r.Kind, rest = ApplicationMetrics, segs[8:] + default: + return Route{}, notFound + } + + r.Namespace = segs[4] + if errs := validation.IsDNS1123Label(r.Namespace); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid namespace %q: %s", r.Namespace, strings.Join(errs, "; ")) + } + if r.Kind == ApplicationLog || r.Kind == ApplicationMetrics { + r.Plural, r.Name = segs[5], segs[6] + if errs := validation.IsDNS1123Label(r.Plural); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid resource %q: %s", r.Plural, strings.Join(errs, "; ")) + } + if errs := validation.IsDNS1123Subdomain(r.Name); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid name %q: %s", r.Name, strings.Join(errs, "; ")) + } + } + r.Endpoint = strings.Join(rest, "/") + return r, nil +} + +// ResourceAttributes is the single SubjectAccessReview a route requires. It +// depends on the path only, so a POST query authorizes exactly like a GET. +func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { + switch r.Kind { + case NamespaceLogs: + return authorizationv1.ResourceAttributes{Verb: "list", Group: coreGroup, Resource: "logs", Namespace: r.Namespace} + case NamespaceMetrics: + return authorizationv1.ResourceAttributes{Verb: "list", Group: coreGroup, Resource: "metrics", Namespace: r.Namespace} + case ApplicationLog: + return authorizationv1.ResourceAttributes{Verb: "get", Group: appsGroup, Resource: r.Plural, Subresource: "log", Name: r.Name, Namespace: r.Namespace} + case ApplicationMetrics: + return authorizationv1.ResourceAttributes{Verb: "get", Group: appsGroup, Resource: r.Plural, Subresource: "metrics", Name: r.Name, Namespace: r.Namespace} + } + return authorizationv1.ResourceAttributes{} +} diff --git a/internal/observabilityproxy/route_test.go b/internal/observabilityproxy/route_test.go new file mode 100644 index 000000000..f34311e0f --- /dev/null +++ b/internal/observabilityproxy/route_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "testing" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +func TestParseRoute(t *testing.T) { + tests := []struct { + name string + path string + wantCode int + want Route + }{ + { + name: "namespace logs", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/logsql/query", + want: Route{Kind: NamespaceLogs, Namespace: "tenant-a", Endpoint: "select/logsql/query"}, + }, + { + name: "namespace logs root without endpoint", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", + want: Route{Kind: NamespaceLogs, Namespace: "tenant-a"}, + }, + { + name: "namespace metrics", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", + want: Route{Kind: NamespaceMetrics, Namespace: "tenant-a", Endpoint: "api/v1/query"}, + }, + { + name: "application log", + path: "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db.main/log/select/logsql/tail", + want: Route{Kind: ApplicationLog, Namespace: "tenant-a", Plural: "postgreses", Name: "db.main", Endpoint: "select/logsql/tail"}, + }, + { + name: "application metrics", + path: "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range", + want: Route{Kind: ApplicationMetrics, Namespace: "tenant-a", Plural: "postgreses", Name: "db", Endpoint: "api/v1/query_range"}, + }, + {name: "unknown group", path: "/apis/core.cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusNotFound}, + {name: "unknown version", path: "/apis/cozystack.io/v1/namespaces/a/logs/x", wantCode: http.StatusNotFound}, + {name: "unknown namespace resource", path: "/apis/cozystack.io/v1alpha1/namespaces/a/events/x", wantCode: http.StatusNotFound}, + {name: "unknown app subresource", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/db/exec/x", wantCode: http.StatusNotFound}, + {name: "core api", path: "/api/v1/namespaces/a/pods", wantCode: http.StatusNotFound}, + {name: "truncated", path: "/apis/cozystack.io/v1alpha1/namespaces/a", wantCode: http.StatusNotFound}, + {name: "empty segment", path: "/apis/cozystack.io/v1alpha1/namespaces/a/logs/select//logsql/query", wantCode: http.StatusBadRequest}, + {name: "leading double slash", path: "//apis/cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot dot segment", path: "/apis/cozystack.io/v1alpha1/namespaces/b/../a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot dot under root", path: "/apis/cozystack.io/v1alpha1/namespaces/a/logs/../metrics/x", wantCode: http.StatusBadRequest}, + {name: "dot dot in unknown prefix", path: "/foo/../apis/cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot segment", path: "/apis/cozystack.io/v1alpha1/namespaces/a/./logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded slash", path: "/apis/cozystack.io/v1alpha1/namespaces/a%2Fb/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded slash lowercase", path: "/apis/cozystack.io/v1alpha1/namespaces/a%2fb/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded dot dot", path: "/apis/cozystack.io/v1alpha1/namespaces/b/%2e%2e/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded dot dot uppercase", path: "/apis/cozystack.io/v1alpha1/namespaces/b/%2E%2E/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "backslash", path: `/apis/cozystack.io/v1alpha1/namespaces/b\..\a/logs/x`, wantCode: http.StatusBadRequest}, + {name: "uppercase namespace", path: "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/x", wantCode: http.StatusBadRequest}, + {name: "namespace with dot", path: "/apis/cozystack.io/v1alpha1/namespaces/a.b/logs/x", wantCode: http.StatusBadRequest}, + {name: "namespace with quote", path: `/apis/cozystack.io/v1alpha1/namespaces/a"b/logs/x`, wantCode: http.StatusBadRequest}, + {name: "name with logsql metacharacters", path: `/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/db"|union/log/x`, wantCode: http.StatusBadRequest}, + {name: "name with braces", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/{db}/log/x", wantCode: http.StatusBadRequest}, + {name: "plural uppercase", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/Postgreses/db/log/x", wantCode: http.StatusBadRequest}, + {name: "plural with dot", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgres.es/db/log/x", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRoute(tt.path) + if tt.wantCode != 0 { + if err == nil { + t.Fatalf("ParseRoute(%q) = %+v, want error with code %d", tt.path, got, tt.wantCode) + } + if err.Code != tt.wantCode { + t.Fatalf("ParseRoute(%q) code = %d (%s), want %d", tt.path, err.Code, err.Message, tt.wantCode) + } + return + } + if err != nil { + t.Fatalf("ParseRoute(%q) error = %v", tt.path, err) + } + if got != tt.want { + t.Fatalf("ParseRoute(%q) = %+v, want %+v", tt.path, got, tt.want) + } + }) + } +} + +func TestRouteResourceAttributes(t *testing.T) { + tests := []struct { + route Route + want authorizationv1.ResourceAttributes + }{ + { + route: Route{Kind: NamespaceLogs, Namespace: "a"}, + want: authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "logs", Namespace: "a"}, + }, + { + route: Route{Kind: NamespaceMetrics, Namespace: "a"}, + want: authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "a"}, + }, + { + route: Route{Kind: ApplicationLog, Namespace: "a", Plural: "postgreses", Name: "db"}, + want: authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db", Namespace: "a"}, + }, + { + route: Route{Kind: ApplicationMetrics, Namespace: "a", Plural: "postgreses", Name: "db"}, + want: authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db", Namespace: "a"}, + }, + } + for _, tt := range tests { + if got := tt.route.ResourceAttributes(); got != tt.want { + t.Errorf("%v.ResourceAttributes() = %+v, want %+v", tt.route.Kind, got, tt.want) + } + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go new file mode 100644 index 000000000..e980818d9 --- /dev/null +++ b/internal/observabilityproxy/server.go @@ -0,0 +1,388 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// tailChunkBytes bounds the memory one tail holds: a longer line is passed +// on in pieces and flushed once its end arrives. +const tailChunkBytes = 64 << 10 + +// errTailIdle cancels a tail that received nothing for the idle timeout; +// it wraps DeadlineExceeded so a store that never answered reads as 504. +var errTailIdle = fmt.Errorf("tail idle: %w", context.DeadlineExceeded) + +// errAborted ends a response whose status line already went out by +// cutting the connection, so a client sees a broken body instead of a +// complete one with lines missing. +var errAborted = &HTTPError{Code: http.StatusBadGateway, Message: "response aborted"} + +// maxFormBytes bounds a POST body; LogsQL queries are a few KiB at most. +const maxFormBytes = 1 << 20 + +// Config is the runtime configuration of the proxy handler. +type Config struct { + ClusterDomain string + // UpstreamURL maps a store namespace to the vlselect base URL; nil + // means the in-cluster Service of the "generic" logs storage. + UpstreamURL func(store string) string + Logs LogsLimits + Limiter LimiterConfig + RequestTimeout time.Duration + TailIdleTimeout time.Duration +} + +// StoreLookup resolves a request namespace to the namespace of its store. +type StoreLookup interface { + Resolve(namespace string) (string, error) +} + +// Server serves the tenant observability roots. +type Server struct { + cfg Config + auth *Authorizer + stores StoreLookup + client *http.Client + limits *limiter + now func() time.Time + requests *prometheus.CounterVec +} + +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { + if cfg.UpstreamURL == nil { + domain := cfg.ClusterDomain + cfg.UpstreamURL = func(store string) string { + return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain) + } + } + s := &Server{ + cfg: cfg, + auth: auth, + stores: stores, + // No client timeout: it would cut live tails. Non-tail requests are + // bounded by a context deadline instead. + client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, + limits: newLimiter(cfg.Limiter), + now: time.Now, + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "observability_proxy_requests_total", + Help: "Requests served, by route kind and status code.", + }, []string{"route", "code"}), + } + if reg != nil { + reg.MustRegister(s.requests, prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "observability_proxy_open_tails", + Help: "Live tails currently open.", + }, func() float64 { return float64(s.limits.openTails()) })) + } + return s +} + +type statusRecorder struct { + http.ResponseWriter + code int + // bodyRead is set once readForm has read the whole body. + bodyRead bool +} + +func (r *statusRecorder) WriteHeader(code int) { + if r.code == 0 { + r.code = code + } + r.ResponseWriter.WriteHeader(code) +} + +func (r *statusRecorder) Write(b []byte) (int, error) { + if r.code == 0 { + r.code = http.StatusOK + } + return r.ResponseWriter.Write(b) +} + +func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter } + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rec := &statusRecorder{ResponseWriter: w} + // The listener has no ReadTimeout, which would cut tails, so the body + // is bounded from here; readForm lifts the deadline once it has read + // the body. A refusal sent before that leaves the deadline in place. + if s.cfg.RequestTimeout > 0 { + _ = http.NewResponseController(w).SetReadDeadline(time.Now().Add(s.cfg.RequestTimeout)) + } + route, herr := ParseRoute(r.URL.EscapedPath()) + if herr == nil { + herr = s.serve(rec, r, route) + } + if herr != nil && herr != errAborted { + // net/http drains up to 256 KiB of an unread body before it sends + // the headers; closing the connection instead answers a refusal at + // once rather than after a body that may never come. + if r.ContentLength != 0 && !rec.bodyRead { + rec.Header().Set("Connection", "close") + } + writeError(rec, herr) + } + s.requests.WithLabelValues(route.Kind.String(), strconv.Itoa(rec.code)).Inc() + if herr == errAborted { + panic(http.ErrAbortHandler) + } +} + +func writeError(w http.ResponseWriter, e *HTTPError) { + if e.Code == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Bearer realm="cozystack"`) + } + http.Error(w, e.Message, e.Code) +} + +// serve answers a root it has no handler for before authenticating, so an +// unknown path costs no review calls. +func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { + if route.Kind != NamespaceLogs { + return errorf(http.StatusNotFound, "not found") + } + user, herr := s.authorize(r, route) + if herr != nil { + return herr + } + if r.Method != http.MethodGet && r.Method != http.MethodPost { + w.Header().Set("Allow", "GET, POST") + return errorf(http.StatusMethodNotAllowed, "method not allowed") + } + if wait, ok := s.limits.allowRate(user); !ok { + w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) + return errorf(http.StatusTooManyRequests, "request rate limit exceeded") + } + endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") + if !ok { + return errorf(http.StatusNotFound, "not found") + } + tail := endpoint == "tail" + acquire := s.limits.acquire + if tail { + acquire = s.limits.acquireTail + } + release, ok := acquire(user) + if !ok { + return tooManyConcurrent(w) + } + defer release() + if herr := s.readForm(w, r); herr != nil { + return herr + } + params, herr := BuildLogsParams(endpoint, r.Form, route.Namespace, s.now(), s.cfg.Logs) + if herr != nil { + return herr + } + + store, err := s.stores.Resolve(route.Namespace) + if err != nil { + slog.Error("resolving log store", "namespace", route.Namespace, "error", err) + return errorf(http.StatusServiceUnavailable, "cannot resolve the log store") + } + + ctx, cancel := context.WithCancelCause(r.Context()) + defer cancel(nil) + limit := s.cfg.RequestTimeout + var idle *time.Timer + if tail { + // Started before the request, so a store that never sends the + // headers is bounded as well as one that stops sending lines. + if limit = s.cfg.TailIdleTimeout; limit > 0 { + idle = time.AfterFunc(limit, func() { cancel(errTailIdle) }) + defer idle.Stop() + } + } else if limit > 0 { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, limit) + defer stop() + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode())) + if err != nil { + return errorf(http.StatusInternalServerError, "building upstream request: %v", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("AccountID", "0") + req.Header.Set("ProjectID", "0") + if tail { + // vlselect holds even the headers of a gzip-encoded tail until the + // first line; an explicit value stops the transport from asking for gzip. + req.Header.Set("Accept-Encoding", "identity") + } + resp, err := s.client.Do(req) + switch { + case err == nil: + case r.Context().Err() != nil: + return errorf(http.StatusBadGateway, "client went away") + case errors.Is(context.Cause(ctx), context.DeadlineExceeded): + return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit) + default: + slog.Warn("log store unreachable", "store", store, "error", err) + return errorf(http.StatusBadGateway, "log store unreachable") + } + defer func() { _ = resp.Body.Close() }() + + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + // Cancelling the upstream request does not interrupt a write to a + // client that stopped reading; only a write deadline does. A tail + // renews it before every line, so only a stalled client is cut; the + // server clears it once the response is finished. + rc := http.NewResponseController(w) + arm := func() {} + if tail && s.cfg.TailIdleTimeout > 0 { + arm = func() { _ = rc.SetWriteDeadline(time.Now().Add(s.cfg.TailIdleTimeout)) } + } else if deadline, ok := ctx.Deadline(); ok { + _ = rc.SetWriteDeadline(deadline) + } + arm() + w.WriteHeader(resp.StatusCode) + if rc.Flush() != nil { + return nil + } + if tail { + s.streamLines(w, rc, resp.Body, idle, arm) + } else if _, err := io.Copy(w, resp.Body); err != nil { + slog.Debug("copying upstream response", "error", err) + return errAborted + } + return nil +} + +// concurrencyRetryAfter is a hint only: a slot frees when another request +// of the user ends, which the proxy cannot predict, and one second keeps +// a polling client from retrying in a tight loop. +const concurrencyRetryAfter = "1" + +func tooManyConcurrent(w http.ResponseWriter) *HTTPError { + w.Header().Set("Retry-After", concurrencyRetryAfter) + return errorf(http.StatusTooManyRequests, "too many concurrent requests") +} + +// readForm reads the whole body under the request timeout, so a client +// that sends it slowly holds its slot only that long. ParseForm leaves the +// body of a GET or of a POST that is not a form unread, and net/http +// drains a leftover body with no deadline once the headers go out, so the +// rest is read here as well; draining rather than refusing a declared +// body keeps any client that sends one working. Once the body is read the +// deadline is lifted: the server's background read would take a later +// expiry for a client that went away and cancel a tail. After a failed +// read it stays, so the server's drain of the unread body is bounded too. +func (s *Server) readForm(w http.ResponseWriter, r *http.Request) *HTTPError { + r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes) + rc := http.NewResponseController(w) + if s.cfg.RequestTimeout > 0 { + _ = rc.SetReadDeadline(time.Now().Add(s.cfg.RequestTimeout)) + } + err := r.ParseForm() + if err == nil { + _, err = io.Copy(io.Discard, r.Body) + } + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + return errorf(http.StatusRequestEntityTooLarge, "request body larger than %d bytes", tooLarge.Limit) + case err == nil: + _ = rc.SetReadDeadline(time.Time{}) + if rec, ok := w.(*statusRecorder); ok { + rec.bodyRead = true + } + return nil + case errors.Is(err, os.ErrDeadlineExceeded): + return errorf(http.StatusRequestTimeout, "request body not received within %s", s.cfg.RequestTimeout) + default: + return errorf(http.StatusBadRequest, "invalid form: %v", err) + } +} + +func (s *Server) authorize(r *http.Request, route Route) (string, *HTTPError) { + scheme, token, _ := strings.Cut(r.Header.Get("Authorization"), " ") + if !strings.EqualFold(scheme, "Bearer") { + return "", errorf(http.StatusUnauthorized, "missing bearer token") + } + // The kube client sets no timeout of its own, and net/http cancels + // r.Context on a dropped client only once the body is read, so a + // stalled review would otherwise outlive every other limit. + ctx := r.Context() + if s.cfg.RequestTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout) + defer cancel() + } + user, err := s.auth.Authenticate(ctx, strings.TrimSpace(token)) + if errors.Is(err, ErrUnauthenticated) { + return "", errorf(http.StatusUnauthorized, "invalid bearer token") + } + if err != nil { + slog.Error("token review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authentication unavailable") + } + allowed, err := s.auth.Authorize(ctx, user, route.ResourceAttributes()) + if err != nil { + slog.Error("subject access review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") + } + if !allowed { + return "", errorf(http.StatusForbidden, "forbidden") + } + return user.Username, nil +} + +// streamLines flushes after every line so a live tail reaches the browser +// as it happens; each line pushes back idle, which cancels the upstream +// stream once no line has arrived for the idle timeout. +func (s *Server) streamLines(w http.ResponseWriter, rc *http.ResponseController, body io.Reader, idle *time.Timer, arm func()) { + rd := bufio.NewReaderSize(body, tailChunkBytes) + for { + chunk, err := rd.ReadSlice('\n') + if len(chunk) > 0 { + if idle != nil { + idle.Reset(s.cfg.TailIdleTimeout) + } + arm() + if _, werr := w.Write(chunk); werr != nil { + return + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + if ferr := rc.Flush(); ferr != nil { + return + } + } + if err != nil && !errors.Is(err, bufio.ErrBufferFull) { + return + } + } +} diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go new file mode 100644 index 000000000..7415b46fa --- /dev/null +++ b/internal/observabilityproxy/server_deadline_test.go @@ -0,0 +1,446 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// pipeListener hands the server one end of a net.Pipe per dial. A pipe has +// no buffer, so a client that stops reading blocks the server's next write +// at once, as a full TCP window eventually does. +type pipeListener struct { + conns chan net.Conn + closed chan struct{} + once sync.Once +} + +type pipeAddr struct{} + +func (pipeAddr) Network() string { return "pipe" } +func (pipeAddr) String() string { return "pipe" } + +func servePipe(t *testing.T, handler http.Handler) *pipeListener { + t.Helper() + l := &pipeListener{conns: make(chan net.Conn), closed: make(chan struct{})} + srv := &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second} + go func() { _ = srv.Serve(l) }() + t.Cleanup(func() { _ = srv.Close() }) + return l +} + +func (l *pipeListener) Accept() (net.Conn, error) { + select { + case c := <-l.conns: + return c, nil + case <-l.closed: + return nil, net.ErrClosed + } +} + +func (l *pipeListener) Close() error { + l.once.Do(func() { close(l.closed) }) + return nil +} + +func (l *pipeListener) Addr() net.Addr { return pipeAddr{} } + +// send opens a connection and writes raw to it without waiting for the +// server to read all of it. +func (l *pipeListener) send(t *testing.T, raw string) net.Conn { + t.Helper() + client, server := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + select { + case l.conns <- server: + case <-time.After(2 * time.Second): + t.Fatal("pipe server is not accepting") + } + go func() { _, _ = client.Write([]byte(raw)) }() + return client +} + +// slotsHeld counts the concurrency and tail slots in use. Tests watch it +// instead of probing with requests of their own, which would compete for +// the slot under test. +func slotsHeld(h *harness) int { + h.server.limits.mu.Lock() + defer h.server.limits.mu.Unlock() + n := h.server.limits.total + for _, c := range h.server.limits.active { + n += c + } + return n +} + +func eventually(t *testing.T, within time.Duration, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(within) + for !cond() { + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(10 * time.Millisecond) + } +} + +func readResponse(t *testing.T, conn net.Conn) *http.Response { + t.Helper() + got := make(chan *http.Response, 1) + go func() { + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + close(got) + return + } + got <- resp + }() + select { + case resp, ok := <-got: + if !ok { + t.Fatal("reading the response failed") + } + return resp + case <-time.After(3 * time.Second): + t.Fatal("no response") + } + return nil +} + +// atUpstream calls probe each time a request reaches the upstream and +// sends the result on the returned channel. The proxy holds the request's +// slot from before it calls the upstream until the upstream answers, so +// the probe sees it held however late the test goroutine gets scheduled. +func atUpstream(probe func(*harness) int, next http.HandlerFunc) (http.HandlerFunc, *atomic.Pointer[harness], <-chan int) { + var h atomic.Pointer[harness] + seen := make(chan int, 1) + return func(w http.ResponseWriter, r *http.Request) { + seen <- probe(h.Load()) + next(w, r) + }, &h, seen +} + +func receive(t *testing.T, ch <-chan int, msg string) int { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(3 * time.Second): + t.Fatal(msg) + } + return 0 +} + +// TestServerSlowBodyHoldsSlotUntilDeadline sends a POST whose body never +// completes: it must count against the user's concurrency, and the read +// must give up at the request timeout instead of holding the handler. +func TestServerSlowBodyHoldsSlotUntilDeadline(t *testing.T) { + // The slot is only observable while the body is pending, so the + // timeout is long enough for the poll below to see it on a busy runner. + h := newHarness(t, Config{ + RequestTimeout: time.Second, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, nil) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "POST "+logsRoot+"query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=") + + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, + "a request with an unfinished body does not hold a concurrency slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode) + } + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "the slot of an unfinished body was not released") +} + +// TestServerStalledClientReleasesTail holds a tail whose client stops +// reading, once before the headers and once after the first line: the +// slot must come back within a few idle timeouts in both cases. +func TestServerStalledClientReleasesTail(t *testing.T) { + for _, readHeaders := range []bool{false, true} { + lines := make(chan string, 1) + done := make(chan struct{}) + upstream, hp, held := atUpstream(func(h *harness) int { return h.server.limits.openTails() }, tailUpstream(lines, done)) + h := newHarness(t, Config{ + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxTails: 1}, + }, upstream) + hp.Store(h) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "GET "+logsRoot+"tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + if readHeaders { + if resp := readResponse(t, conn); resp.StatusCode != http.StatusOK { + t.Fatalf("tail answered %d", resp.StatusCode) + } + lines <- `{"_msg":"unread"}` + } + if n := receive(t, held, "the tail never reached the store"); n != 1 { + t.Fatalf("open tails while the store answers = %d, want 1", n) + } + eventually(t, 2*time.Second, func() bool { return h.server.limits.openTails() == 0 }, + "a client that stopped reading keeps its tail slot") + } +} + +// TestServerStalledClientReleasesQuery does the same for a non-tail +// response, which must be written by the request deadline. +func TestServerStalledClientReleasesQuery(t *testing.T) { + upstream, hp, held := atUpstream(slotsHeld, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/stream+json") + _, _ = io.WriteString(w, `{"_msg":"hello"}`+"\n") + }) + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, upstream) + hp.Store(h) + pipe := servePipe(t, h.server) + pipe.send(t, "GET "+logsRoot+"query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + if n := receive(t, held, "the stalled request never reached the store"); n != 1 { + t.Fatalf("slots held while the store answers = %d, want 1", n) + } + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "a client that stopped reading keeps its concurrency slot") +} + +// TestServerActiveTailOutlivesWriteDeadline keeps a tail busy for several +// idle timeouts: the write deadline must be renewed, not fixed. +func TestServerActiveTailOutlivesWriteDeadline(t *testing.T) { + lines := make(chan string) + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{TailIdleTimeout: 150 * time.Millisecond}, tailUpstream(lines, done)) + resp := h.do(http.MethodGet, logsRoot+"tail?query=*", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + rd := bufio.NewReader(resp.Body) + for i := range 8 { + lines <- `{"_msg":"line"}` + if _, err := rd.ReadString('\n'); err != nil { + t.Fatalf("line %d: %v", i, err) + } + time.Sleep(75 * time.Millisecond) + } +} + +// TestServerTailUpstreamWithoutHeaders gives up on a store that accepts a +// tail but never answers it, instead of holding the slot until the client +// leaves. +func TestServerTailUpstreamWithoutHeaders(t *testing.T) { + release := make(chan struct{}) + h := newHarness(t, Config{ + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxTails: 1}, + }, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + // After the harness, so it runs before the servers wait for handlers. + t.Cleanup(func() { close(release) }) + got := make(chan int, 1) + go func() { got <- h.status(http.MethodGet, logsRoot+"tail?query=*", "good") }() + select { + case code := <-got: + if code != http.StatusGatewayTimeout { + t.Fatalf("silent store answered %d, want 504", code) + } + case <-time.After(2 * time.Second): + t.Fatal("a tail waits for upstream headers without bound") + } + if n := h.server.limits.openTails(); n != 0 { + t.Fatalf("open tails = %d after the tail ended", n) + } +} + +// TestServerAbortsTruncatedResponse cuts the client connection when the +// store's body breaks off after the headers went out, so a partial answer +// does not end as a clean 200. +func TestServerAbortsTruncatedResponse(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"_msg":"a"}`+"\n") + }) + resp := h.do(http.MethodGet, logsRoot+"query?query=*", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("a truncated store answer reached the client as a complete response") + } +} + +// TestServerUndeliveredBodyReleasesSlot declares a body that never comes +// on requests whose form parsing does not read it: a GET, a POST that is +// not a form, and a GET tail. net/http would drain that body with no +// deadline once the headers go out, so the slot must still come back. +func TestServerUndeliveredBodyReleasesSlot(t *testing.T) { + for name, raw := range map[string]string{ + "get query": "GET " + logsRoot + "query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 10\r\n\r\n", + "post text query": "POST " + logsRoot + "query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Type: text/plain\r\nContent-Length: 10\r\n\r\n", + "get tail": "GET " + logsRoot + "tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 10\r\n\r\n", + } { + t.Run(name, func(t *testing.T) { + lines := make(chan string, 1) + done := make(chan struct{}) + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}, + }, tailUpstream(lines, done)) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + conn := pipe.send(t, raw) + held := func() bool { return slotsHeld(h) > 0 } + eventually(t, time.Second, held, "the request never took a slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("undelivered body answered %d, want 408", resp.StatusCode) + } + eventually(t, time.Second, func() bool { return !held() }, "an undelivered body keeps its slot") + }) + } +} + +func TestServerOversizedBody(t *testing.T) { + h := newHarness(t, Config{}, nil) + for _, ct := range []string{"application/x-www-form-urlencoded", "text/plain"} { + req, _ := http.NewRequest(http.MethodPost, h.proxy.URL+logsRoot+"query?query=*", strings.NewReader("query="+strings.Repeat("x", maxFormBytes))) + req.Header.Set("Authorization", "Bearer good") + req.Header.Set("Content-Type", ct) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Errorf("%s body over the cap answered %d, want 413", ct, resp.StatusCode) + } + } +} + +// TestServerEarlyRefusalWithUnreadBody refuses requests before the body is +// read: a bad token, a full concurrency slot, and a full tail slot. +// net/http drains up to 256 KiB of an unread body before it sends the +// headers, so the answer must not wait on a body that never comes, even +// under a long request timeout. +func TestServerEarlyRefusalWithUnreadBody(t *testing.T) { + for _, timeout := range []time.Duration{200 * time.Millisecond, time.Minute} { + for _, tc := range []struct { + name, raw string + hold string + want int + }{ + {"bad token", "POST " + logsRoot + "query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer bad\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 1000\r\n\r\nquery=", "", http.StatusUnauthorized}, + {"concurrency slot taken", "POST " + logsRoot + "query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 1000\r\n\r\nquery=", "query", http.StatusTooManyRequests}, + {"tail slot taken", "GET " + logsRoot + "tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 1000\r\n\r\n", "tail", http.StatusTooManyRequests}, + } { + // A held query ends at the request timeout by design, so the + // slot is only reliably taken under the long one. + if tc.hold == "query" && timeout < time.Second { + continue + } + t.Run(fmt.Sprintf("%s/%s", timeout, tc.name), func(t *testing.T) { + lines := make(chan string, 1) + done := make(chan struct{}) + h := newHarness(t, Config{ + RequestTimeout: timeout, + TailIdleTimeout: time.Minute, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}, + }, tailUpstream(lines, done)) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + if tc.hold != "" { + pipe.send(t, "GET "+logsRoot+tc.hold+"?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the first request never took its slot") + } + start := time.Now() + conn := pipe.send(t, tc.raw) + resp := readResponse(t, conn) + if resp.StatusCode != tc.want || time.Since(start) > time.Second { + t.Fatalf("answered %d after %s, want %d within a second", resp.StatusCode, time.Since(start), tc.want) + } + }) + } + } +} + +// stalledAPIServer answers the TokenReview for "good" unless stallTokens +// is set, and never answers a SubjectAccessReview: it holds each stalled +// call until the caller gives up or the test ends. +func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface { + t.Helper() + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/tokenreviews") && !stallTokens { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"apiVersion":"authentication.k8s.io/v1","kind":"TokenReview","status":{"authenticated":true,"user":{"username":"alice"}}}`) + return + } + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(srv.Close) + // After srv, so it runs first and srv.Close does not wait on a stalled call. + t.Cleanup(func() { close(release) }) + return kubernetes.NewForConfigOrDie(&rest.Config{Host: srv.URL}) +} + +// TestServerStalledReviewAnswersInTime holds TokenReview or +// SubjectAccessReview open on a POST whose body is complete but unread: +// net/http does not watch the connection until the body is read, so only +// a bounded review call ends the request. +func TestServerStalledReviewAnswersInTime(t *testing.T) { + for _, tc := range []struct { + name string + stallTokens bool + }{{"token review", true}, {"subject access review", false}} { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) + h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second) + got := make(chan int, 1) + go func() { + resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil) + _ = resp.Body.Close() + got <- resp.StatusCode + }() + select { + case code := <-got: + if code != http.StatusServiceUnavailable { + t.Fatalf("stalled %s answered %d, want 503", tc.name, code) + } + case <-time.After(2 * time.Second): + t.Fatalf("a stalled %s holds the request past ten request timeouts", tc.name) + } + }) + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go new file mode 100644 index 000000000..df3e23bc4 --- /dev/null +++ b/internal/observabilityproxy/server_test.go @@ -0,0 +1,496 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +type upstreamRequest struct { + path string + form url.Values + headers http.Header +} + +type harness struct { + t *testing.T + kube *fakeKube + proxy *httptest.Server + server *Server + mu sync.Mutex + got []upstreamRequest + stores []string + upstream *httptest.Server +} + +type mapStores map[string]string + +func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } + +// newHarness wires the proxy to a fake apiserver that knows tokens "good" +// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only, +// and to an upstream served by handler. +func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { + h := &harness{t: t} + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return a.Namespace == "tenant-a" && a.Resource == "logs" + }) + h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("upstream ParseForm: %v", err) + } + h.mu.Lock() + h.got = append(h.got, upstreamRequest{path: r.URL.Path, form: r.PostForm, headers: r.Header.Clone()}) + h.mu.Unlock() + if handler != nil { + handler(w, r) + return + } + w.Header().Set("Content-Type", "application/stream+json") + _, _ = io.WriteString(w, `{"_msg":"hello"}`+"\n") + })) + t.Cleanup(h.upstream.Close) + cfg.UpstreamURL = func(store string) string { + h.mu.Lock() + h.stores = append(h.stores, store) + h.mu.Unlock() + return h.upstream.URL + } + if cfg.Logs == (LogsLimits{}) { + cfg.Logs = testLimits() + } + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server.now = func() time.Time { return testNow } + h.proxy = httptest.NewServer(h.server) + t.Cleanup(h.proxy.Close) + return h +} + +const logsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/logsql/" + +func (h *harness) do(method, path, token string, body url.Values, headers map[string]string) *http.Response { + h.t.Helper() + var rd io.Reader + if body != nil { + rd = strings.NewReader(body.Encode()) + } + req, err := http.NewRequest(method, h.proxy.URL+path, rd) + if err != nil { + h.t.Fatal(err) + } + if body != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + h.t.Fatal(err) + } + return resp +} + +func (h *harness) status(method, path, token string) int { + h.t.Helper() + resp := h.do(method, path, token, nil, nil) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode +} + +func (h *harness) requests() []upstreamRequest { + h.mu.Lock() + defer h.mu.Unlock() + return append([]upstreamRequest(nil), h.got...) +} + +func TestServerStatusCodes(t *testing.T) { + h := newHarness(t, Config{}, nil) + tests := []struct { + name, method, path, token string + want int + }{ + {"ok", http.MethodGet, logsRoot + "query?query=error", "good", http.StatusOK}, + {"missing token", http.MethodGet, logsRoot + "query", "", http.StatusUnauthorized}, + {"invalid token", http.MethodGet, logsRoot + "query", "bad", http.StatusUnauthorized}, + {"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden}, + {"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound}, + {"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound}, + {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, + {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, + {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, + {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, + {"ignore_global_time_filter", http.MethodGet, logsRoot + "query?query=" + url.QueryEscape("options(ignore_global_time_filter=true) *"), "good", http.StatusBadRequest}, + {"inverted range", http.MethodGet, logsRoot + "query?start=2026-11-04T11:00:00Z&end=2026-11-04T10:00:00Z", "good", http.StatusBadRequest}, + {"dot dot reaching another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/../tenant-a/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"encoded dot dot", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/%2e%2e/tenant-a/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"encoded slash", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a%2Flogs/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"double slash", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs//select/logsql/query", "good", http.StatusBadRequest}, + {"delete", http.MethodDelete, logsRoot + "query", "good", http.StatusMethodNotAllowed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := h.status(tt.method, tt.path, tt.token); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + }) + } + if reqs := h.requests(); len(reqs) != 1 { + t.Fatalf("upstream saw %d requests, want only the allowed one", len(reqs)) + } +} + +func TestServerRebuildsUpstreamRequest(t *testing.T) { + h := newHarness(t, Config{}, nil) + hostile := url.Values{ + "query": {"error"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{kubernetes_namespace_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret; other=1"} + + resp := h.do(http.MethodGet, logsRoot+"query?"+hostile.Encode(), "good", nil, headers) + _ = resp.Body.Close() + resp = h.do(http.MethodPost, logsRoot+"query?extra_filters="+url.QueryEscape(`{x="y"}`), "good", hostile, headers) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != `{"_msg":"hello"}`+"\n" { + t.Fatalf("POST answered %d %q", resp.StatusCode, body) + } + if ct := resp.Header.Get("Content-Type"); ct != "application/stream+json" { + t.Fatalf("Content-Type = %q, want the upstream's", ct) + } + + reqs := h.requests() + if len(reqs) != 2 { + t.Fatalf("upstream saw %d requests", len(reqs)) + } + if reqs[0].form.Encode() != reqs[1].form.Encode() { + t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form) + } + if n := h.kube.sars.Load(); n != 1 { + t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n) + } + for _, r := range reqs { + if r.path != "/select/logsql/query" { + t.Errorf("upstream path = %s", r.path) + } + if got := r.form["extra_stream_filters"]; len(got) != 1 || got[0] != wantScope { + t.Errorf("extra_stream_filters = %q", got) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") { + t.Errorf("client scope parameter forwarded: %v", r.form) + } + if r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("tenant headers = %q/%q, want 0/0", r.headers.Get("AccountID"), r.headers.Get("ProjectID")) + } + if ae := r.headers.Get("Accept-Encoding"); !strings.Contains(ae, "gzip") { + t.Errorf("upstream query Accept-Encoding = %q, want gzip outside tails", ae) + } + for _, k := range []string{"Authorization", "X-Forwarded-For", "Cookie"} { + if r.headers.Get(k) != "" { + t.Errorf("client header %s forwarded", k) + } + } + } + for _, s := range h.stores { + if s != "tenant-a-store" { + t.Errorf("store = %s, want the resolved one", s) + } + } +} + +func TestServerUpstreamErrors(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "cannot parse query", http.StatusBadRequest) + }) + if got := h.status(http.MethodGet, logsRoot+"query?query=|", "good"); got != http.StatusBadRequest { + t.Fatalf("upstream 400 answered as %d", got) + } + h.upstream.Close() + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusBadGateway { + t.Fatalf("unreachable upstream answered %d, want 502", got) + } +} + +func TestServerLimits(t *testing.T) { + release := make(chan struct{}) + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + }) + first := h.do(http.MethodGet, logsRoot+"query", "good", nil, nil) + defer func() { _ = first.Body.Close() }() + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusTooManyRequests { + t.Fatalf("concurrent request answered %d, want 429", got) + } + once.Do(func() { close(release) }) + + h2 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, nil) + if got := h2.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusOK { + t.Fatalf("first request answered %d", got) + } + limited := h2.do(http.MethodGet, logsRoot+"query", "good", nil, nil) + _ = limited.Body.Close() + if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" { + t.Fatalf("rate-limited request answered %d with Retry-After %q, want 429 with 1000", + limited.StatusCode, limited.Header.Get("Retry-After")) + } +} + +// tailUpstream sends one line per value received on lines and holds the +// stream open until done is closed. +func tailUpstream(lines <-chan string, done <-chan struct{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/stream+json") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + for { + select { + case l := <-lines: + _, _ = io.WriteString(w, l+"\n") + w.(http.Flusher).Flush() + case <-done: + return + case <-r.Context().Done(): + return + } + } + } +} + +func TestServerTailFlushesEveryLine(t *testing.T) { + lines := make(chan string) + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{TailIdleTimeout: time.Minute, RequestTimeout: 50 * time.Millisecond}, tailUpstream(lines, done)) + + resp := h.do(http.MethodGet, logsRoot+"tail?query=*&since=2026-11-04T08:00:00Z", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + rd := bufio.NewReader(resp.Body) + for i, l := range []string{`{"_msg":"one"}`, `{"_msg":"two"}`} { + select { + case lines <- l: + case <-time.After(2 * time.Second): + t.Fatalf("upstream tail closed before line %d", i) + } + got := make(chan string, 1) + go func() { + s, _ := rd.ReadString('\n') + got <- s + }() + select { + case s := <-got: + if s != l+"\n" { + t.Fatalf("line %d = %q", i, s) + } + case <-time.After(2 * time.Second): + t.Fatalf("line %d not flushed to the client", i) + } + // Past the request timeout: a tail must not be cut by it. + time.Sleep(60 * time.Millisecond) + } + if r := h.requests()[0]; r.path != "/select/logsql/tail" || r.form.Get("start_offset") != "900000ms" { + t.Fatalf("tail upstream = %s %v, want start_offset capped at 15m", r.path, r.form) + } +} + +// TestServerTailHeadersBeforeFirstLine mimics vlselect, which holds even the +// response headers of a gzip-negotiated tail until the first line arrives. +func TestServerTailHeadersBeforeFirstLine(t *testing.T) { + done := make(chan struct{}) + h := newHarness(t, Config{TailIdleTimeout: time.Minute}, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + select { + case <-done: + case <-r.Context().Done(): + } + return + } + tailUpstream(nil, done)(w, r) + }) + // Registered after the harness so it runs before the servers close and + // releases a held upstream when the assertion below fails. + t.Cleanup(func() { close(done) }) + + req, err := http.NewRequest(http.MethodGet, h.proxy.URL+logsRoot+"tail?query=*", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer good") + got := make(chan *http.Response, 1) + go func() { + resp, err := http.DefaultClient.Do(req) + if err != nil { + close(got) + return + } + got <- resp + }() + select { + case resp, ok := <-got: + if !ok { + t.Fatal("tail request failed") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + case <-time.After(2 * time.Second): + t.Fatal("tail response headers did not reach the client before the first line") + } + reqs := h.requests() + if len(reqs) != 1 { + t.Fatalf("upstream saw %d requests, want 1", len(reqs)) + } + if ae := reqs[0].headers.Get("Accept-Encoding"); ae != "identity" { + t.Fatalf("upstream tail Accept-Encoding = %q, want identity", ae) + } +} + +func TestServerTailCapsAndIdle(t *testing.T) { + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{ + Limiter: LimiterConfig{MaxTailsPerUser: 1, MaxTails: 2}, + TailIdleTimeout: 500 * time.Millisecond, + }, tailUpstream(make(chan string), done)) + + a := h.do(http.MethodGet, logsRoot+"tail", "good", nil, nil) + defer func() { _ = a.Body.Close() }() + if a.StatusCode != http.StatusOK { + t.Fatalf("first tail answered %d", a.StatusCode) + } + if got := h.status(http.MethodGet, logsRoot+"tail", "good"); got != http.StatusTooManyRequests { + t.Fatalf("second tail of one user answered %d, want 429", got) + } + b := h.do(http.MethodGet, logsRoot+"tail", "bob-token", nil, nil) + defer func() { _ = b.Body.Close() }() + if b.StatusCode != http.StatusOK { + t.Fatalf("bob's tail answered %d", b.StatusCode) + } + if got := h.server.limits.openTails(); got != 2 { + t.Fatalf("open tails = %d, want 2", got) + } + + closed := make(chan error, 1) + go func() { + _, err := io.Copy(io.Discard, a.Body) + closed <- err + }() + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("idle tail not closed") + } + deadline := time.Now().Add(2 * time.Second) + for h.server.limits.openTails() > 1 { + if time.Now().After(deadline) { + t.Fatal("closed tail still holds its slot") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestServerUpstreamDeadline(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + h := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("slow upstream answered %d, want 504", got) + } +} + +func TestServerTailLongLine(t *testing.T) { + long := `{"_msg":"` + strings.Repeat("x", 3*tailChunkBytes) + `"}` + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, long+"\n"+`{"_msg":"short"}`+"\n") + }) + resp := h.do(http.MethodGet, logsRoot+"tail", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if string(body) != long+"\n"+`{"_msg":"short"}`+"\n" { + t.Fatalf("long line not streamed intact: got %d bytes", len(body)) + } +} + +// The auth scheme is case-insensitive (RFC 9110, section 11.1), as +// kube-apiserver reads it. +func TestServerBearerSchemeCase(t *testing.T) { + h := newHarness(t, Config{}, nil) + for _, auth := range []string{"Bearer good", "bearer good", "BEARER good"} { + resp := h.do(http.MethodGet, logsRoot+"query", "", nil, map[string]string{"Authorization": auth}) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("Authorization %q answered %d, want 200", auth, resp.StatusCode) + } + } + for _, auth := range []string{"Basic good", "Bearer", "Bearer ", "Bearergood"} { + resp := h.do(http.MethodGet, logsRoot+"query", "", nil, map[string]string{"Authorization": auth}) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("Authorization %q answered %d, want 401", auth, resp.StatusCode) + } + } +} + +// A 429 for concurrency tells the client when to retry, as the rate 429 +// does, so console polls back off instead of retrying at once. +func TestServerConcurrencyRetryAfter(t *testing.T) { + release := make(chan struct{}) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}}, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + }) + t.Cleanup(func() { close(release) }) + for _, endpoint := range []string{"query", "tail"} { + first := h.do(http.MethodGet, logsRoot+endpoint, "good", nil, nil) + defer func() { _ = first.Body.Close() }() + second := h.do(http.MethodGet, logsRoot+endpoint, "good", nil, nil) + _ = second.Body.Close() + if second.StatusCode != http.StatusTooManyRequests || second.Header.Get("Retry-After") != "1" { + t.Errorf("%s over the limit answered %d with Retry-After %q, want 429 with 1", + endpoint, second.StatusCode, second.Header.Get("Retry-After")) + } + } +} diff --git a/internal/observabilityproxy/store.go b/internal/observabilityproxy/store.go new file mode 100644 index 000000000..f24f0b17b --- /dev/null +++ b/internal/observabilityproxy/store.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/validation" + corev1listers "k8s.io/client-go/listers/core/v1" +) + +// monitoringLabel is the label the tenant chart sets on every tenant +// namespace and the collector routes logs by +// (packages/system/monitoring-agents/templates/_tenant-log-routes.tpl). +const monitoringLabel = "namespace.cozystack.io/monitoring" + +// StoreResolver maps a namespace to the tenant namespace whose store holds +// its logs, by the same rule the collector writes with. +type StoreResolver struct { + namespaces corev1listers.NamespaceLister + synced func() bool + root string +} + +// NewStoreResolver reads namespaces from a cache that synced reports as +// filled. +func NewStoreResolver(namespaces corev1listers.NamespaceLister, synced func() bool, rootTenant string) *StoreResolver { + return &StoreResolver{namespaces: namespaces, synced: synced, root: rootTenant} +} + +// Resolve answers the root tenant for a namespace that no longer exists, +// because its label went with it. Lines it wrote while routed to another +// tenant's store are not found there; reading them needs routing history +// the proxy does not keep. +func (r *StoreResolver) Resolve(namespace string) (string, error) { + // Before the first list every namespace looks gone, which would send + // a labeled one to the root store; readiness alone does not stop a + // client that addresses the pod directly. + if !r.synced() { + return "", errors.New("namespace cache not synced") + } + ns, err := r.namespaces.Get(namespace) + if apierrors.IsNotFound(err) { + return r.root, nil + } + if err != nil { + return "", err + } + target := ns.Labels[monitoringLabel] + if target == "" { + return r.root, nil + } + // The value becomes a DNS name; a dot in it would address a host in + // some other namespace. + if errs := validation.IsDNS1123Label(target); len(errs) > 0 { + return "", fmt.Errorf("namespace %s: invalid %s label %q: %s", namespace, monitoringLabel, target, strings.Join(errs, "; ")) + } + return target, nil +} diff --git a/internal/observabilityproxy/store_test.go b/internal/observabilityproxy/store_test.go new file mode 100644 index 000000000..d21476b98 --- /dev/null +++ b/internal/observabilityproxy/store_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" +) + +// The cases mirror packages/system/monitoring-agents/tests/ +// fluent-bit_tenant_log_routing_test.yaml: reads must resolve to the store +// the collector wrote to. +func TestStoreResolver(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + add := func(name string, labels map[string]string) { + if err := indexer.Add(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}}); err != nil { + t.Fatal(err) + } + } + add("cozy-system", nil) + add("tenant-empty", map[string]string{monitoringLabel: ""}) + for ns, target := range map[string]string{ + "tenant-root": "tenant-root", + "tenant-inherits-root": "tenant-root", + "tenant-a": "tenant-a", + "tenant-a-child": "tenant-a", + "tenant-b": "tenant-b", + } { + add(ns, map[string]string{monitoringLabel: target}) + } + add("tenant-forged", map[string]string{monitoringLabel: "pod.tenant-b"}) + lister := corev1listers.NewNamespaceLister(indexer) + synced := func() bool { return true } + if got, err := NewStoreResolver(lister, synced, "tenant-root").Resolve("tenant-forged"); err == nil { + t.Fatalf("label naming a host outside a namespace resolved to %q", got) + } + + tests := []struct { + root, ns, want string + }{ + {"tenant-root", "cozy-system", "tenant-root"}, + {"tenant-root", "tenant-empty", "tenant-root"}, + {"tenant-root", "tenant-root", "tenant-root"}, + {"tenant-root", "tenant-inherits-root", "tenant-root"}, + {"tenant-root", "tenant-a", "tenant-a"}, + {"tenant-root", "tenant-a-child", "tenant-a"}, + {"tenant-root", "tenant-b", "tenant-b"}, + {"tenant-root", "tenant-gone", "tenant-root"}, + {"tenant-a", "tenant-a", "tenant-a"}, + {"tenant-a", "tenant-a-child", "tenant-a"}, + {"tenant-a", "tenant-root", "tenant-root"}, + {"tenant-a", "tenant-inherits-root", "tenant-root"}, + {"tenant-a", "cozy-system", "tenant-a"}, + } + for _, tt := range tests { + r := NewStoreResolver(lister, synced, tt.root) + got, err := r.Resolve(tt.ns) + if err != nil { + t.Fatalf("root=%s Resolve(%s) error: %v", tt.root, tt.ns, err) + } + if got != tt.want { + t.Errorf("root=%s Resolve(%s) = %s, want %s", tt.root, tt.ns, got, tt.want) + } + } +} + +// TestStoreResolverUnsynced: before the first list, every namespace looks +// absent, and an absent one resolves to the root store; a labeled +// namespace would read the wrong store. +func TestStoreResolverUnsynced(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + r := NewStoreResolver(corev1listers.NewNamespaceLister(indexer), func() bool { return false }, "tenant-root") + if got, err := r.Resolve("tenant-a"); err == nil { + t.Fatalf("unsynced cache resolved tenant-a to %q", got) + } +} diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 827b5d579..bc439d40b 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -222,18 +222,35 @@ spec: key: backup - name: ALLOW_EMPTY_BACKUPS value: "true" + {{- /* + create_remote keeps the local backup when its upload fails, + and the upstream defaults never prune local backups (0) and + resume unfinished uploads at start, which pushes a failed + attempt to the bucket as an archive no Backup refers to and + holds the API lock (423) while it does. -1 keeps only the + newest local backup and removes it once its upload succeeds. + */}} + - name: BACKUPS_TO_KEEP_LOCAL + value: "-1" + - name: API_COMPLETE_RESUMABLE_AFTER_RESTART + value: "false" - name: REMOTE_STORAGE value: s3 {{- /* + On the useSystemBucket flow every S3 setting but the + release's own S3_PATH below, the platform's + backupStorage.forcePathStyle included, comes + from the clickhouse-backup.yml key the platform projects + into cozy-backups-creds, mounted below as the config file. + clickhouse-backup reloads that file in each API handler + that reaches S3 and kubelet rewrites a mounted Secret in + place, so credentials projected after the Pod started, or + rotated since, reach the running sidecar; environment + would be read only at start and would override the file, + so none of those is set here. + S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3 - addressing: SeaweedFS (the platform default) needs - path-style, while AWS public S3 needs vhost-style in many - regions. On the useSystemBucket flow the value is the - platform's backupStorage.forcePathStyle, which this app - chart cannot read directly, so it is projected into - cozy-backups-creds and consumed via an optional - secretKeyRef (optional: true so the Pod starts before the - first projection). On the legacy paths (own Secret via + addressing. On the legacy paths (own Secret via s3CredentialsSecret.name, or a configured endpoint) we cannot peek inside the Secret at render time, so we keep the historical path-style default — AWS users in that mode @@ -241,12 +258,8 @@ spec: clickhouse-backup config. */}} {{- if .Values.backup.useSystemBucket }} - - name: S3_FORCE_PATH_STYLE - valueFrom: - secretKeyRef: - name: {{ $sidecarSecretName | quote }} - key: forcePathStyle - optional: true + - name: CLICKHOUSE_BACKUP_CONFIG + value: /etc/clickhouse-backup-s3/config.yml {{- else if or (not (eq (default "" .Values.backup.s3CredentialsSecret.name) "")) .Values.backup.endpoint }} - name: S3_FORCE_PATH_STYLE value: "true" @@ -282,44 +295,18 @@ spec: */}} - name: S3_PATH value: {{ $s3Path | quote }} - {{- /* - optional: true in the useSystemBucket path so the Pod - starts even before the platform projector materialises - cozy-backups-creds in this namespace (projection runs at - first BackupJob time; the sidecar surfaces backup errors - only when its HTTP API is exercised). Without this gate - every fresh ClickHouse release with useSystemBucket=true - would hang in CreateContainerConfigError until a - BackupJob fires. Legacy path keeps the strict reference - shape — its Secret is chart-emitted and present at - install time. - */}} - {{- $secretOptional := .Values.backup.useSystemBucket }} + {{- if not .Values.backup.useSystemBucket }} - name: S3_BUCKET - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } - {{- /* - clickhouse-backup's aws-sdk-go-v2 client rejects a scheme-less - S3_ENDPOINT ("was not a valid URI"). On the useSystemBucket flow - the projected cozy-backups-creds.endpoint is a bare host, so - prepend https:// (the platform's CNPG endpoint helper does the - same). Legacy / BYO-Secret endpoints are tenant-supplied with - their own scheme and are consumed verbatim. - */}} - {{- if .Values.backup.useSystemBucket }} - - name: S3_ENDPOINT_HOST - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }}, optional: true } } - - name: S3_ENDPOINT - value: "https://$(S3_ENDPOINT_HOST)" - {{- else }} + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }} } } - name: S3_ENDPOINT valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }} } } - {{- end }} - name: S3_REGION - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }} } } - name: S3_ACCESS_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }} } } - name: S3_SECRET_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }} } } + {{- end }} {{- if $endpointCAName }} # Trust the mounted private-CA bundle for the S3 endpoint (see # the `backup.endpointCA` field in values.yaml). SSL_CERT_DIR, @@ -343,6 +330,11 @@ spec: - name: backup-s3-ca mountPath: /etc/ssl/cozy-s3-ca readOnly: true +{{- end }} +{{- if .Values.backup.useSystemBucket }} + - name: backup-s3-config + mountPath: /etc/clickhouse-backup-s3 + readOnly: true {{- end }} securityContext: # clickhouse-server stores data under /var/lib/clickhouse with @@ -366,13 +358,14 @@ spec: runAsGroup: 0 seccompProfile: type: RuntimeDefault -{{- if $endpointCAName }} - # Private-CA bundle for the backup sidecar, projected as ca.crt for - # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds - # the whole ClickHouse Pod in ContainerCreating (naming the Secret in - # `kubectl describe`) rather than mounting an empty trust dir and - # failing later inside clickhouse-backup with an opaque x509 error. +{{- if or $endpointCAName .Values.backup.useSystemBucket }} volumes: +{{- if $endpointCAName }} + # Private-CA bundle for the backup sidecar, projected as ca.crt for + # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds + # the whole ClickHouse Pod in ContainerCreating (naming the Secret in + # `kubectl describe`) rather than mounting an empty trust dir and + # failing later inside clickhouse-backup with an opaque x509 error. - name: backup-s3-ca secret: secretName: {{ $endpointCAName | quote }} @@ -380,6 +373,18 @@ spec: - key: {{ $endpointCAKey | quote }} path: ca.crt {{- end }} +{{- if .Values.backup.useSystemBucket }} + # Optional: the platform projects cozy-backups-creds into this + # namespace at the first BackupJob, after the Pod has started. + - name: backup-s3-config + secret: + secretName: {{ $sidecarSecretName | quote }} + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml +{{- end }} +{{- end }} {{- end }} serviceTemplates: - name: svc-template diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 66eda4dc8..a01e10ea5 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -2,10 +2,12 @@ suite: backup sidecar S3 wiring (useSystemBucket vs legacy, path-override guard) # Locks in the clickhouse-backup sidecar S3 wiring this PR introduced on the # ClickHouseInstallation PodTemplate: -# - useSystemBucket=true points every S3_* env at the platform-projected -# cozy-backups-creds Secret with optional:true (the Pod starts before the -# first projection) and scopes S3_PATH to /; the -# chart-managed -backup-s3 / -backup Secrets are skipped; +# - useSystemBucket=true mounts the clickhouse-backup.yml key of the +# platform-projected cozy-backups-creds Secret as the sidecar's config +# file with optional:true (the Pod starts before the first projection), +# sets no other S3 setting in the environment, and scopes S3_PATH to +# /; the chart-managed -backup-s3 / +# -backup Secrets are skipped; # - the legacy flow keeps the chart-emitted -backup-s3 Secret and a # strict (non-optional) secretKeyRef; # - s3PathOverride + useSystemBucket fails fast at render time. @@ -25,7 +27,7 @@ set: _cluster: {} tests: - - it: "useSystemBucket=true: sidecar reads cozy-backups-creds (optional) and scopes S3_PATH" + - it: "useSystemBucket=true: sidecar reads its S3 settings from the projected config file and scopes S3_PATH" template: templates/clickhouse.yaml documentSelector: path: kind @@ -45,28 +47,30 @@ tests: - equal: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_PATH")].value value: tenant-test/ch-test + # clickhouse-backup reloads this file in each API handler that reaches + # S3 and kubelet rewrites the mounted Secret in place, so credentials + # projected after the Pod started, or rotated since, reach the running + # sidecar. - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name - value: cozy-backups-creds + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="CLICKHOUSE_BACKUP_CONFIG")].value + value: /etc/clickhouse-backup-s3/config.yml - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.optional + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].mountPath + value: /etc/clickhouse-backup-s3 + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].readOnly value: true - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - # The projected endpoint is a bare host; clickhouse-backup needs a URI, so - # the chart prepends https:// via env-var expansion on this flow. - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.key - value: endpoint - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")].value - value: https://$(S3_ENDPOINT_HOST) + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret + value: + secretName: cozy-backups-creds + # The Pod starts before the first projection. + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml - - it: "useSystemBucket=true: S3_FORCE_PATH_STYLE is sourced from cozy-backups-creds, not hardcoded" + - it: "useSystemBucket=true: no S3 setting is fixed in the sidecar environment" template: templates/clickhouse.yaml documentSelector: path: kind @@ -76,18 +80,21 @@ tests: enabled: true useSystemBucket: true asserts: - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.key - value: forcePathStyle - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.optional - value: true + # Environment overrides the config file and is read only at start. - notExists: - # must no longer be the hardcoded literal value: "true" - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].value + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_REGION")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ACCESS_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")] - it: "useSystemBucket=true: chart-managed backup Secrets are skipped" set: @@ -240,7 +247,10 @@ tests: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="SSL_CERT_DIR")].value value: /etc/ssl/cozy-s3-ca:/etc/ssl/certs - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-ca")].secret.secretName + value: my-s3-ca + - equal: + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret.secretName value: cozy-backups-creds # backup.enabled=false renders no sidecar at all, so the CA must not leak a @@ -292,3 +302,42 @@ tests: - notMatchRegex: path: stringData["backup.sh"] pattern: 'echo DB_LIST=' + + # A strategy attempt that fails mid-upload leaves its local backup behind, + # and with the upstream defaults the sidecar would keep it forever and push + # it to the bucket on its next start as an archive no Backup refers to. The + # sidecar must neither resume uploads at start nor keep local copies once an + # upload succeeds, on either S3 flow. + - it: "sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: true + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" + + - it: "legacy flow: sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: false + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 17e2a2822..2ac5cfc45 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -87,7 +87,7 @@ See the reference for components utilized in this service: - **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk. -- **Air-gapped tenant workers (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos OS disk image fetched over HTTP by CDI and a `.../installer/...` installer reference. Both sources are now overridable — set `talos.imageFactoryURL` (default `https://factory.talos.dev`) to a self-hosted Image Factory, a caching mirror, or an internal HTTP file server, and `talos.installerRepository` (default `factory.talos.dev/installer`) to a mirrored registry. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig and was removed in this release; **in-guest container-image pulls** (pods pulling from registries inside the tenant cluster) still honour no per-tenant `registries.mirrors` override until Phase 2 restores it via `machine.registries.mirrors` knobs. So a strict-egress tenant can now mirror the Talos OS image and installer, but rate-limited in-guest registry pulls remain a Phase 2 follow-up — file an issue if you depend on this and it is not yet landed. +- **Air-gapped tenant workers (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos OS disk image fetched over HTTP by CDI and a `.../installer/...` installer reference. Both sources are now overridable — set `talos.imageFactoryURL` (default `https://factory.talos.dev`) to a self-hosted Image Factory, a caching mirror, or an internal HTTP file server, and `talos.installerRepository` (default `factory.talos.dev/installer`) to a mirrored registry. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig, so the Secret is removed together with the platform `registries` value that fed it. A Package that still sets `registries` keeps installing, the key is ignored. Registry mirrors for tenant workers, **in-guest container-image pulls** (pods pulling from registries inside the tenant cluster) included, are configured per worker pool through `talos.registryMirrors` of the `KubernetesNodes` resource. - **Worker MachineHealthCheck remediation is now enabled by default** (Phase 1 Talos rollover): the worker MHC `maxUnhealthy` moved from a hard-coded `0` (remediation effectively **off**) to a configurable `nodeHealthCheck.maxUnhealthy` defaulting to `"50%"`. CAPI now auto-remediates (deletes and replaces) unhealthy worker Machines while at least 50% of a group stays healthy. `"50%"` deliberately leaves headroom for transient unhealthy nodes during the kubeadm→Talos rollover and slow first boots from `factory.talos.dev`; set `nodeHealthCheck.maxUnhealthy: "0%"` to keep the previous remediation-off behaviour until your fleet is stable on Talos workers. diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index 9e3fd3ee5..662016692 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -45,7 +45,8 @@ spec: cluster: {{ .Release.Name }} tenant: {{ .Release.Namespace }} remoteWrite: - url: http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus + urls: + - http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus fluent-bit: readinessProbe: httpGet: diff --git a/packages/apps/kubernetes/templates/ingress.yaml b/packages/apps/kubernetes/templates/ingress.yaml index 9b35f9683..9b88a7fce 100644 --- a/packages/apps/kubernetes/templates/ingress.yaml +++ b/packages/apps/kubernetes/templates/ingress.yaml @@ -3,7 +3,7 @@ on an etcd-less tenant the chart collapses to the awaiting-etcd beacon, so no addon controller exists and the Service would select nothing. */}} {{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd (eq .Values.addons.ingressNginx.exposeMethod "Proxied") }} -{{- if .Values.addons.ingressNginx.hosts }} +{{- if and .Values.addons.ingressNginx.hosts $ingress }} --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml b/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml index 9583b16a3..75e9349a8 100644 --- a/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml +++ b/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml @@ -79,3 +79,20 @@ tests: asserts: - hasDocuments: count: 0 + + # An empty _namespace.ingress means no ingress-nginx serves this tenant, and + # the API rejects the ingressClassName "" the Ingress would carry. + - it: renders the Service but no Ingress when the tenant has no ingress + set: + _namespace: + ingress: "" + addons: + ingressNginx: + enabled: true + exposeMethod: Proxied + hosts: ["app.example.com"] + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service diff --git a/packages/apps/kubernetes/tests/monitoring_agents_test.yaml b/packages/apps/kubernetes/tests/monitoring_agents_test.yaml new file mode 100644 index 000000000..1dc21181a --- /dev/null +++ b/packages/apps/kubernetes/tests/monitoring_agents_test.yaml @@ -0,0 +1,42 @@ +suite: monitoring-agents HelmRelease values match the monitoring-agents chart + +# The values below are consumed by packages/system/monitoring-agents, which +# reads vmagent.remoteWrite.urls as a list and the tenant label from +# vmagent.externalLabels.tenant. A key the chart does not read is dropped in +# silence, and the chart then falls back to cozy-monitoring defaults that do +# not resolve inside a tenant cluster. Its counterpart is +# packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml. +templates: + - templates/helmreleases/monitoring-agents.yaml + +values: + - values/common.yaml + +release: + name: kubernetes-clusterprod + namespace: tenant-a + +set: + addons.monitoringAgents.enabled: true + _namespace.monitoring: tenant-root + _cluster: + cluster-domain: cozy.local + +tests: + - it: sends metrics to the shortterm vminsert of the monitoring tenant as a urls list + asserts: + - equal: + path: spec.values.vmagent.remoteWrite + value: + urls: + - http://vminsert-shortterm.tenant-root.svc.cozy.local:8480/insert/0/prometheus + + - it: labels series with the tenant that owns the cluster, not the monitoring tenant + asserts: + - equal: + path: spec.values.vmagent.externalLabels + value: + cluster: kubernetes-clusterprod + tenant: tenant-a + - notExists: + path: spec.values.global diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 50c300aff..c508476f4 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -279,13 +279,13 @@ For `sslmode=verify-full` to work, the CA bundle retrieved above must be saved t ### Databases configuration -| Name | Description | Type | Value | -| -------------------------------- | ---------------------------------------- | ------------------- | ----- | -| `databases` | Databases configuration map. | `map[string]object` | `{}` | -| `databases[name].roles` | Roles assigned to users. | `object` | `{}` | -| `databases[name].roles.admin` | List of users with admin privileges. | `[]string` | `[]` | -| `databases[name].roles.readonly` | List of users with read-only privileges. | `[]string` | `[]` | -| `databases[name].extensions` | List of enabled PostgreSQL extensions. | `[]string` | `[]` | +| Name | Description | Type | Value | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ------------------- | ----- | +| `databases` | Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. | `map[string]object` | `{}` | +| `databases[name].roles` | Roles assigned to users. | `object` | `{}` | +| `databases[name].roles.admin` | List of users with admin privileges. | `[]string` | `[]` | +| `databases[name].roles.readonly` | List of users with read-only privileges. | `[]string` | `[]` | +| `databases[name].extensions` | List of enabled PostgreSQL extensions. | `[]string` | `[]` | ### Backup parameters diff --git a/packages/apps/postgres/templates/init-script.yaml b/packages/apps/postgres/templates/init-script.yaml index 395267ff0..6306a5fae 100644 --- a/packages/apps/postgres/templates/init-script.yaml +++ b/packages/apps/postgres/templates/init-script.yaml @@ -53,6 +53,9 @@ PostgreSQL identifiers too, capped at 63. */}} {{- range $database, $d := .Values.databases }} + {{- if has $database (list "postgres" "template0" "template1") }} + {{- fail (printf "databases.%s is refused: %q is a PostgreSQL system database. Managing it would hand it, its public schema and the auto-grant event trigger to the \"%s_admin\" role (template1 then passes all of that on to every database created afterwards), and removing the entry later could not drop it. Remove the entry and declare an application database instead; the next upgrade releases a system database a previous version took over, without dropping it" $database $database $database) }} + {{- end }} {{- if or (not (regexMatch "^[-._a-zA-Z0-9]+$" $database)) (gt (len $database) 54) }} {{- fail (printf "invalid database name %q: a database name may contain only letters, digits and the characters -._ and must be at most 54 characters (it is interpolated into superuser SQL and the init-script shell, and it seeds the \"_admin\" and \"_readonly\" PostgreSQL roles, which CREATE ROLE silently truncates past 63 bytes; the next init-job run then cannot drop the truncated role and fails). Changing only its key in values makes the init-job drop the database; to keep the data, a platform administrator first renames it as the CNPG superuser (ALTER DATABASE, and ALTER ROLE for its _admin and _readonly roles), then the key is changed" $database) }} {{- end }} @@ -320,6 +323,67 @@ stringData: EOT done + echo "== release system databases" + {{- /* + Versions that did not refuse postgres, template0 and template1 in + databases took them over like any other entry. DROP DATABASE on them can + only fail (template databases cannot be dropped, and this session is + connected to postgres), which wedged every later upgrade, so they are + handed back instead: the auto-grant trigger goes, since it names a role + about to be dropped and would make every CREATE SCHEMA fail; what the + _admin and _readonly roles own there goes to cozystack:orphaned, + as in the orphaned-roles pass and for the same reason (a tenant's + SECURITY DEFINER function must not end up owned by the superuser), and is + made SECURITY INVOKER (only a role marked "cozystack:managed" is touched: + a role that merely carries one of those names was not created by this + chart and keeps what it owns); the database itself returns to the superuser, + public to pg_database_owner (to the superuser before PostgreSQL 15), each + only when one of those marked roles owns it and before the reassignment + would move it (or when cozystack:orphaned does, after a run that stopped + in between); an owner the chart did not mark keeps what it owns, and + the comment is cleared. The orphaned-roles pass below then drops the + roles; it does not visit template1, hence the per-database pass here. + template0 accepts no connection, and the Job that commented it never got + further, so only the comment is cleared. A database created from + template1 while it was taken over inherited its trigger, naming + template1_admin, and its public schema owned by that role; the trigger + goes, public returns to its default owner if a marked template1_admin + still owns it, and a declared database gets + its trigger back from the grant pass further down. + */}} + SYSTEM_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm' AND d.datname IN ('postgres', 'template0', 'template1')") + echo "system databases to release: $SYSTEM_DBS" + for db in $SYSTEM_DBS; do + if [ "$db" != template0 ]; then + psql -v ON_ERROR_STOP=1 --echo-all --dbname "$db" -v db="$db" <<\EOT + DROP EVENT TRIGGER IF EXISTS trigger_auto_grant; + DROP FUNCTION IF EXISTS auto_grant_schema_privileges(); + SELECT format('ALTER SCHEMA public OWNER TO %s', CASE WHEN current_setting('server_version_num')::int >= 150000 THEN 'pg_database_owner' ELSE 'CURRENT_USER' END) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND (o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\gexec + SELECT format('ALTER DATABASE %I OWNER TO CURRENT_USER', d.datname) FROM pg_database d JOIN pg_roles o ON o.oid = d.datdba WHERE d.datname = :'db' AND (o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\gexec + SELECT format('REASSIGN OWNED BY %I TO "cozystack:orphaned"', r), format('DROP OWNED BY %I', r) FROM unnest(ARRAY[:'db' || '_admin', :'db' || '_readonly']) AS r WHERE EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid JOIN pg_roles u ON u.oid = m.member WHERE g.rolname = 'cozystack:managed' AND u.rolname = r)\gexec + SELECT format('ALTER %s %s SECURITY INVOKER', CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.oid::regprocedure) FROM pg_proc p WHERE p.proowner = '"cozystack:orphaned"'::regrole AND p.prosecdef\gexec + SELECT format('ALTER VIEW %s SET (security_invoker = true)', c.oid::regclass) FROM pg_class c WHERE c.relowner = '"cozystack:orphaned"'::regrole AND c.relkind = 'v' AND current_setting('server_version_num')::int >= 150000 AND NOT coalesce('security_invoker=true' = ANY (c.reloptions), false)\gexec + EOT + fi + if [ "$db" = template1 ]; then + {{- /* + An assignment of its own, so that set -e stops the pass when the lookup + fails: inside the for list its status is discarded, the comment below is + cleared without a clone visited, and no later run selects template1 again. + */}} + CLONE_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate") + for other in $CLONE_DBS; do + psql -v ON_ERROR_STOP=1 --echo-queries --dbname "$other" <<\EOT + SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant', 'DROP FUNCTION auto_grant_schema_privileges()' WHERE position('''template1_admin''' IN coalesce((SELECT prosrc FROM pg_proc WHERE oid = to_regprocedure('auto_grant_schema_privileges()')), '')) > 0\gexec + SELECT format('ALTER SCHEMA public OWNER TO %s', CASE WHEN current_setting('server_version_num')::int >= 150000 THEN 'pg_database_owner' ELSE 'CURRENT_USER' END) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid)\gexec + EOT + done + fi + psql -v ON_ERROR_STOP=1 --echo-all -v db="$db" <<\EOT + COMMENT ON DATABASE :"db" IS NULL; + EOT + done + echo "== delete databases" {{- /* A restore replaces spec.databases with the Backup's snapshot, but recovery @@ -338,7 +402,7 @@ stringData: ADOPTED_MARK=$(psql -v ON_ERROR_STOP=1 -t -A --dbname postgres -c "SELECT current_setting('cozystack.adopted_restore', true)") MANAGED_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm'") DEFINED_DBS="{{ join " " (keys .Values.databases | sortAlpha) }}" - DELETE_DBS=$(for db in $MANAGED_DBS; do case " $DEFINED_DBS " in *" $db "*) :;; *) echo $db;; esac; done) + DELETE_DBS=$(for db in $MANAGED_DBS; do case " postgres template0 template1 $DEFINED_DBS " in *" $db "*) :;; *) echo $db;; esac; done) if [ -n "$RESTORE_MARK" ] && [ "$ADOPTED_MARK" != "$RESTORE_MARK" ]; then echo "databases the restore brought back that values do not declare, released from management: $DELETE_DBS" diff --git a/packages/apps/postgres/tests/system_databases_test.yaml b/packages/apps/postgres/tests/system_databases_test.yaml new file mode 100644 index 000000000..e4ff8400f --- /dev/null +++ b/packages/apps/postgres/tests/system_databases_test.yaml @@ -0,0 +1,135 @@ +suite: init-script never takes over or drops a system database + +# Each of postgres, template0 and template1 used to be accepted in databases and +# handled like an application database: owned by _admin, given the +# auto-grant event trigger, and on removal sent to DROP DATABASE, which fails on +# all three (template databases cannot be dropped, and the Job is connected to +# postgres) and wedged every later upgrade. template1 also passed the takeover +# on to every database created afterwards. + +templates: + - templates/init-script.yaml + +release: + name: pg-test + namespace: tenant-test + +set: + _cluster: + scheduling: {} + +tests: + - it: refuses postgres in databases + set: + databases: + postgres: {} + asserts: + - failedTemplate: + errorPattern: "databases.postgres is refused: \"postgres\" is a PostgreSQL system database" + + - it: refuses template0 in databases + set: + databases: + template0: {} + asserts: + - failedTemplate: + errorPattern: "databases.template0 is refused: \"template0\" is a PostgreSQL system database" + + - it: refuses template1 in databases + set: + databases: + template1: {} + asserts: + - failedTemplate: + errorPattern: "databases.template1 is refused: \"template1\" is a PostgreSQL system database" + + - it: still accepts an application database whose name contains a system name + set: + databases: + postgres_app: {} + mytemplate1: {} + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: 'CREATE DATABASE "postgres_app"' + + # A release that a previous version let manage a system database still carries + # its comment. The delete pass must never select it, whatever values say. + - it: never lists a system database for DROP DATABASE + set: + databases: + appdb: {} + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: 'DELETE_DBS=\$\(for db in \$MANAGED_DBS; do case " postgres template0 template1 \$DEFINED_DBS " in \*" \$db "\*\) :;; \*\) echo \$db;; esac; done\)' + + - it: hands a taken-over system database back before the delete pass + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "SYSTEM_DBS=\\$\\(psql -v ON_ERROR_STOP=1 -t -A -c \"SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm' AND d.datname IN \\('postgres', 'template0', 'template1'\\)\"\\)" + - matchRegex: + path: stringData["init.sh"] + pattern: '(?s)== release system databases.*DROP EVENT TRIGGER IF EXISTS trigger_auto_grant;\s+DROP FUNCTION IF EXISTS auto_grant_schema_privileges\(\);.*ALTER SCHEMA public OWNER TO %s.*ALTER DATABASE %I OWNER TO CURRENT_USER.*REASSIGN OWNED BY %I TO "cozystack:orphaned"''.*DROP OWNED BY %I.*COMMENT ON DATABASE :"db" IS NULL;.*== delete databases' + + # A tenant holding _admin may have created a SECURITY DEFINER function + # there; handed to the superuser it would run with superuser rights for + # anyone PUBLIC can execute it. It goes to cozystack:orphaned and is made + # SECURITY INVOKER, as in the orphaned-roles pass. + - it: never hands a released system database's objects to the superuser + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - notMatchRegex: + path: stringData["init.sh"] + pattern: 'REASSIGN OWNED BY %I TO CURRENT_USER' + - matchRegex: + path: stringData["init.sh"] + pattern: '(?s)== release system databases.*ALTER %s %s SECURITY INVOKER.*ALTER VIEW %s SET \(security_invoker = true\).*== delete databases' + + # A database created from template1 while it was taken over inherited the + # trigger naming template1_admin, which the orphaned-roles pass then drops; + # CREATE SCHEMA would fail there from that point on. + - it: drops the template1 trigger inherited by other databases + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)== release system databases.*if \\[ \"\\$db\" = template1 \\]; then.*SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant', 'DROP FUNCTION auto_grant_schema_privileges\\(\\)' WHERE position\\('''template1_admin''' IN .*== delete databases" + + # That clone's public schema also came from template1, owned by + # template1_admin; left to the orphaned-roles pass it would end up owned by + # cozystack:orphaned and the database owner would lose CREATE on it. + - it: gives an inherited public schema back to its default owner + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)if \\[ \"\\$db\" = template1 \\]; then.*SELECT format\\('ALTER SCHEMA public OWNER TO %s', .*\\) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS \\(SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid\\).*== delete databases" + + # template0 accepts no connection: only postgres and template1 are connected + # to and released, template0 just has its comment cleared. The behaviour is + # run against a fake psql in hack/postgres-release-system-databases.bats. + - it: connects to every released system database but template0 + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)for db in \\$SYSTEM_DBS; do\\s+if \\[ \"\\$db\" != template0 \\]; then\\s+psql -v ON_ERROR_STOP=1 --echo-all --dbname \"\\$db\" -v db=\"\\$db\"" diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 91747ba24..4d544ff6b 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -313,7 +313,7 @@ } }, "databases": { - "description": "Databases configuration map.", + "description": "Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.", "type": "object", "default": {}, "additionalProperties": { diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 83b658a64..fb4a67b35 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -287,7 +287,7 @@ users: {} ## @field {DatabaseRoles} [roles] - Roles assigned to users. ## @field {[]string} [extensions] - List of enabled PostgreSQL extensions. -## @param {map[string]Database} databases - Databases configuration map. +## @param {map[string]Database} databases - Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. databases: {} ## Example: ## databases: diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index 2365ca9c1..f2a0d001d 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -48,6 +48,7 @@ virtctl ssh @ | `disks` | List of disks to attach. | `[]object` | `[]` | | `disks[i].name` | Disk name. | `string` | `""` | | `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `disks[i].bootOrder` | Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. | `*int` | `null` | | `networks` | Networks to attach the VM to. | `[]object` | `[]` | | `networks[i].name` | Network attachment name. | `string` | `""` | | `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index cfeffa810..b757ab96d 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -30,6 +30,13 @@ spec: {{- if .Values.external }} {{- if and (eq .Values.externalMethod "WholeIP") (not .Values.externalPorts) }} - port: 65535 + {{- else if not .Values.externalPorts }} + {{- /* A Service needs at least one port. cozy-proxy only admits TCP and + UDP ports into its PortList filter, so an SCTP placeholder keeps the + external IP while every inbound TCP and UDP packet is dropped. */}} + - name: no-ports + port: 65535 + protocol: SCTP {{- else }} {{- range .Values.externalPorts }} - name: port-{{ . }} diff --git a/packages/apps/vm-instance/templates/vm.yaml b/packages/apps/vm-instance/templates/vm.yaml index 8153adf8f..0d7845621 100644 --- a/packages/apps/vm-instance/templates/vm.yaml +++ b/packages/apps/vm-instance/templates/vm.yaml @@ -92,7 +92,16 @@ spec: {{- end }} {{- end }} disks: + {{- /* KubeVirt rejects two devices sharing a bootOrder, so a clash is + reported here with the disk names rather than surfacing as a + webhook denial on the VirtualMachine. */}} + {{- $bootOrderOwners := dict }} {{- range $i, $disk := .Values.disks }} + {{- $bootOrder := int ($disk.bootOrder | default (add $i 1)) }} + {{- with get $bootOrderOwners (toString $bootOrder) }} + {{- fail (printf "disks %q and %q both have bootOrder %d; boot orders must be unique" . $disk.name $bootOrder) }} + {{- end }} + {{- $_ := set $bootOrderOwners (toString $bootOrder) $disk.name }} - name: disk-{{ $disk.name }} {{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" $disk.name) }} {{- if $dv }} @@ -104,7 +113,7 @@ spec: {{- else }} {{ $deviceKey }}: {} {{- end }} - bootOrder: {{ add $i 1 }} + bootOrder: {{ $bootOrder }} {{- else }} {{- fail (printf "Specified disk not exists in cluster: %s" .name) }} {{- end }} diff --git a/packages/apps/vm-instance/tests/disk_boot_order_test.yaml b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml new file mode 100644 index 000000000..146cbaf53 --- /dev/null +++ b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml @@ -0,0 +1,128 @@ +suite: vm-instance disk boot order + +# The first entry of `disks` is the one callers treat as the system disk, so +# booting from installer or rescue media must not require moving that media to +# the front of the list. These cases pin that an explicit bootOrder decides +# which device the VM boots from while list order stays as given, and that +# leaving it unset keeps the positional order existing VMs were rendered with. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/vm.yaml +set: + fullnameOverride: test-vm + instanceType: "" + instanceProfile: "" + resources: + cpu: 1 + sockets: 1 + memory: 1Gi + _cluster: + scheduling: + dedicatedNodesForWindowsVMs: "false" +kubernetesProvider: + scheme: + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: + group: cdi.kubevirt.io + version: v1beta1 + resource: datavolumes + namespaced: true + objects: + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-system + namespace: tenant-test + annotations: {} + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-iso + namespace: tenant-test + annotations: + vm-disk.cozystack.io/optical: "true" + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-data + namespace: tenant-test + annotations: {} +tests: + - it: keeps the positional boot order when no disk sets bootOrder + set: + disks: + - name: system + - name: iso + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 1 + - name: disk-iso + cdrom: {} + bootOrder: 2 + - name: disk-data + disk: {} + bootOrder: 3 + + - it: boots from a CD-ROM in second position without reordering the disks + set: + disks: + - name: system + bootOrder: 2 + - name: iso + bus: sata + bootOrder: 1 + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 2 + - name: disk-iso + cdrom: + bus: sata + bootOrder: 1 + - name: disk-data + disk: {} + bootOrder: 3 + - equal: + path: spec.template.spec.volumes[0].dataVolume.name + value: vm-disk-system + + - it: refuses an explicit bootOrder that collides with another disk's positional one + set: + disks: + - name: system + - name: iso + bootOrder: 1 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 1; boot orders must be unique' + + - it: refuses two disks given the same explicit bootOrder + set: + disks: + - name: system + bootOrder: 3 + - name: iso + bootOrder: 3 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 3; boot orders must be unique' + + - it: rejects a bootOrder below 1 through the values schema + set: + disks: + - name: system + bootOrder: 0 + asserts: + - failedTemplate: + errorPattern: "minimum" diff --git a/packages/apps/vm-instance/tests/service_ports_test.yaml b/packages/apps/vm-instance/tests/service_ports_test.yaml new file mode 100644 index 000000000..d7689d957 --- /dev/null +++ b/packages/apps/vm-instance/tests/service_ports_test.yaml @@ -0,0 +1,70 @@ +suite: vm-instance external Service ports + +# The API server rejects a Service without ports, so an external PortList VM +# whose port list was emptied could not be upgraded at all. The placeholder +# must be SCTP: cozy-proxy only admits TCP and UDP ports into its filter, so +# any TCP or UDP placeholder would open that port on the VM. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/service.yaml +set: + fullnameOverride: test-vm + external: true + +tests: + - it: renders an SCTP placeholder for PortList with no ports + set: + externalMethod: PortList + externalPorts: [] + asserts: + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "false" + - lengthEqual: + path: spec.ports + count: 1 + - equal: + path: spec.ports[0] + value: + name: no-ports + port: 65535 + protocol: SCTP + + - it: renders the listed ports for PortList + set: + externalMethod: PortList + externalPorts: [22, 443] + asserts: + - lengthEqual: + path: spec.ports + count: 2 + - equal: + path: spec.ports[0] + value: + name: port-22 + port: 22 + targetPort: 22 + - notContains: + path: spec.ports + content: + name: no-ports + port: 65535 + protocol: SCTP + + - it: keeps the WholeIP placeholder when no ports are listed + set: + externalMethod: WholeIP + externalPorts: [] + asserts: + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "true" + - equal: + path: spec.ports + value: + - port: 65535 diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 7c24a9e52..45adaac72 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -69,6 +69,11 @@ "name" ], "properties": { + "bootOrder": { + "description": "Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.", + "type": "integer", + "minimum": 1 + }, "bus": { "description": "Disk bus type (e.g. \"sata\").", "type": "string" diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index 88fc6f8dd..2730812a0 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -10,6 +10,8 @@ ## @field {string} name - Disk name. ## @x-cozystack-options {source: vmdisk} ## @field {string} [bus] - Disk bus type (e.g. "sata"). +## @field {*int} [bootOrder] - Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. +## @minimum 1 ## @typedef {struct} Network - Network to attach the VM to. ## @field {string} [name] - Network attachment name. @@ -67,6 +69,12 @@ disks: [] ## - name: example-system ## - name: example-data ## bus: sata +## Boot from an attached ISO while keeping the system disk first in the list: +## disks: +## - name: example-system +## bootOrder: 2 +## - name: example-iso +## bootOrder: 1 ## @param {[]Network} networks - Networks to attach the VM to. networks: [] diff --git a/packages/core/platform/sources/observability-proxy.yaml b/packages/core/platform/sources/observability-proxy.yaml new file mode 100644 index 000000000..d9e9ec14b --- /dev/null +++ b/packages/core/platform/sources/observability-proxy.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.observability-proxy +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.networking + components: + - name: observability-proxy + path: system/observability-proxy + install: + namespace: cozy-observability-proxy + releaseName: observability-proxy diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 4a0f0ae89..db6c7109b 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -261,6 +261,10 @@ stringData: null/omitted in custom values. Missing key -> the true default, matching the chart values.yaml. */}} monitoring-enabled: {{ dig "rootEnabled" true (.Values.monitoring | default dict) | quote }} + {{- /* Whether the root tenant runs its own ingress-nginx + (Tenant/root spec.ingress). `dig`, as for monitoring-enabled + above, so an explicit false survives. */}} + ingress-enabled: {{ dig "rootEnabled" true (.Values.ingress | default dict) | quote }} {{- /* The rate each shared-central tracing tenant's collector holds it to. Only written when set: every release in every tenant reads this Secret, and a key appearing would upgrade all of them. */}} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index b175714a2..906c8d95f 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -344,6 +344,7 @@ variants and opt-in on the slim ones. */}} {{include "cozystack.platform.package.full.default" (list "cozystack.metrics-server" $) }} {{- $monitoringAgentsComponents := dict "monitoring-agents" (dict "values" (dict "global" (dict "target" "tenant-root"))) -}} {{include "cozystack.platform.package.full" (list "cozystack.monitoring-agents" "default" $ $monitoringAgentsComponents) }} +{{include "cozystack.platform.package.full.default" (list "cozystack.observability-proxy" $) }} {{include "cozystack.platform.package.full.default" (list "cozystack.goldpinger" $) }} {{include "cozystack.platform.package.default" (list "cozystack.prometheus-operator-crds" $) }} {{include "cozystack.platform.package.full.default" (list "cozystack.grafana-operator" $) }} diff --git a/packages/core/platform/templates/containerd-registry-secret.yaml b/packages/core/platform/templates/containerd-registry-secret.yaml deleted file mode 100644 index 6e9fb6408..000000000 --- a/packages/core/platform/templates/containerd-registry-secret.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if .Values.registries.mirrors }} -apiVersion: v1 -kind: Secret -metadata: - name: patch-containerd - namespace: cozy-system -type: Opaque -stringData: -{{- range $registry, $mirror := .Values.registries.mirrors }} -{{- if $mirror.endpoints }} - {{ $registry }}.toml: | - server = "https://{{ $registry }}" -{{- range $endpoint := $mirror.endpoints }} - [host."{{ $endpoint }}"] - capabilities = ["pull", "resolve"] -{{- $endpointConfig := index $.Values.registries.config $endpoint }} -{{- if $endpointConfig }} -{{- if $endpointConfig.tls }} -{{- if $endpointConfig.tls.insecureSkipVerify }} - skip_verify = true -{{- end }} -{{- end }} -{{- if $endpointConfig.auth }} - [host."{{ $endpoint }}".auth] - username = "{{ $endpointConfig.auth.username }}" - password = "{{ $endpointConfig.auth.password }}" -{{- end }} -{{- else }} - skip_verify = true -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - diff --git a/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml b/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml new file mode 100644 index 000000000..68f6a8168 --- /dev/null +++ b/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml @@ -0,0 +1,35 @@ +suite: apps.yaml ingress.rootEnabled → _cluster wiring +templates: + - templates/apps.yaml + +release: + name: cozy-platform + namespace: cozy-system + +# ingress.rootEnabled records whether the root tenant runs its own +# ingress-nginx (Tenant/root spec.ingress). It reaches cozystack-basics through +# _cluster.ingress-enabled in the cozystack-values Secret, where it decides +# whether tenant-root's _namespace.ingress names tenant-root or stays empty. + +tests: + - it: writes ingress-enabled true by default + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"true"' + + - it: writes ingress-enabled false when ingress.rootEnabled is disabled + set: + ingress.rootEnabled: false + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"false"' + + - it: falls back to true (no panic) when ingress is null + set: + ingress: null + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"true"' diff --git a/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml b/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml index 221aeeb27..6758b1b42 100644 --- a/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml +++ b/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml @@ -42,6 +42,7 @@ bundles: - cozystack.monitoring - cozystack.nats-application - cozystack.nfs-driver + - cozystack.observability-proxy - cozystack.openbao-application - cozystack.opensearch-application - cozystack.opensearch-operator diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml new file mode 100644 index 000000000..9e6ec583d --- /dev/null +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -0,0 +1,69 @@ +suite: observability-proxy package wiring +# The proxy reads the tenant log and metrics stores, which the full variants +# run by default and the slim ones only on request, so it follows +# monitoring-agents: on by default in the full variants, opt-in on the slim. +templates: + - templates/sources.yaml + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: installs into cozy-observability-proxy once networking is up + template: templates/sources.yaml + documentSelector: + path: metadata.name + value: cozystack.observability-proxy + asserts: + - equal: + path: kind + value: PackageSource + - equal: + path: spec.variants[0].dependsOn + value: + - cozystack.networking + - equal: + path: spec.variants[0].components + value: + - name: observability-proxy + path: system/observability-proxy + install: + namespace: cozy-observability-proxy + releaseName: observability-proxy + + - it: emits the package on a full variant + template: templates/bundles/system.yaml + values: + - ../values-isp-full.yaml + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true + + - it: leaves the package out of a slim variant by default + template: templates/bundles/system.yaml + values: + - ../values-isp-slim.yaml + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true + not: true + + - it: emits the package on a slim variant that lists it in enabledPackages + template: templates/bundles/system.yaml + values: + - ../values-isp-slim.yaml + set: + bundles.enabledPackages: + - cozystack.observability-proxy + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2ce252ed..f2e35b5e5 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -669,6 +669,19 @@ gateway: - cozy-monitoring - cozy-linstor-gui - default +# Platform ingress configuration. +ingress: + # Whether the root tenant (tenant-root) runs its own ingress-nginx, i.e. + # Tenant/root spec.ingress=true. It decides whether tenant-root's + # _namespace.ingress names tenant-root or stays empty, which is what + # applications there (SeaweedFS waiting on ingress-nginx-system, the + # ingressClassName of their Ingresses) and every tenant inheriting from the + # root read. Nothing in the charts sets Tenant/root spec.ingress, so keep + # this in sync with it by hand; changing it re-renders cozystack-basics and + # the releases reading the cozystack-values Secret. The default is true only + # to keep the value existing clusters already render. + rootEnabled: true + # Platform monitoring configuration. monitoring: # Whether the root tenant (tenant-root) hosts the platform @@ -768,37 +781,6 @@ scheduling: # loginTheme: acme # loginThemeImage: registry.example.com/acme-keycloak-theme:v1 branding: {} -# Container registry mirrors configuration -# -# Example: -# registries: -# mirrors: -# docker.io: -# endpoints: -# - http://10.0.0.1:8082 -# ghcr.io: -# endpoints: -# - http://10.0.0.1:8083 -# gcr.io: -# endpoints: -# - http://10.0.0.1:8084 -# registry.k8s.io: -# endpoints: -# - http://10.0.0.1:8085 -# quay.io: -# endpoints: -# - http://10.0.0.1:8086 -# cr.fluentbit.io: -# endpoints: -# - http://10.0.0.1:8087 -# docker-registry3.mariadb.com: -# endpoints: -# - http://10.0.0.1:8088 -# config: -# "10.0.0.1:8082": -# tls: -# insecureSkipVerify: true -registries: {} # Resource allocation ratios resources: cpuAllocationRatio: 10 diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml index d04c5b651..4917206de 100644 --- a/packages/extra/ingress/templates/nginx-scrape.yaml +++ b/packages/extra/ingress/templates/nginx-scrape.yaml @@ -9,6 +9,13 @@ spec: matchNames: - {{ .Release.Namespace }} podMetricsEndpoints: + # Most series describe the Ingress objects served, in any namespace, so + # the application identity the vmagent scrape class stamps from the + # controller pod would misattribute them; and with honorLabels an + # exporter-supplied identity would win over the class. The controller's + # own series (nginx process, reload status, go_*, up, scrape_*) lose the + # Ingress application's identity too; that is the accepted cost. The + # target-stage drop covers up and scrape_*, which skip metric relabeling. - port: metrics honorLabels: true interval: 30s @@ -19,6 +26,11 @@ spec: targetLabel: node - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller @@ -46,6 +58,11 @@ spec: targetLabel: controller - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml new file mode 100644 index 000000000..4914049c8 --- /dev/null +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -0,0 +1,62 @@ +suite: ingress-nginx series do not carry an application identity +templates: + - templates/nginx-scrape.yaml +release: + name: ingress + namespace: tenant-root + +# The vmagent scrape class stamps the scraped pod's application identity on +# every series. Most ingress-nginx series describe the Ingress objects it +# serves, in whatever namespace they live (honorLabels keeps their +# namespace), so the controller's identity would misattribute them, and with +# honorLabels true an exporter-supplied identity would win over the class. +# The controller's own health series lose that identity too, by design. Both +# endpoints drop the identity instead: from the target, because vmagent +# adds up and scrape_* with the target labels after metric relabeling, and +# from the scraped samples, because an exporter's copy is not renamed. +tests: + - it: drops the identity on every endpoint + asserts: + - lengthEqual: + path: spec.podMetricsEndpoints + count: 1 + - equal: + path: spec.podMetricsEndpoints[0].metricRelabelConfigs + value: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + + # Endpoint rules run after the scrape class rules, so the drop takes + # effect wherever it sits in this list; it is pinned last so that no + # later rule can set the identity again. + - it: drops the identity from the basic endpoint's target as its last rule + documentIndex: 0 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + + - it: drops the identity from the detailed endpoint's target as its last rule + documentIndex: 1 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + targetLabel: controller + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) diff --git a/packages/extra/seaweedfs/README.md b/packages/extra/seaweedfs/README.md index e269992bf..0540db4af 100644 --- a/packages/extra/seaweedfs/README.md +++ b/packages/extra/seaweedfs/README.md @@ -78,4 +78,7 @@ | `s3.resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | | `s3.resources.memory` | Amount of memory allocated. | `quantity` | `""` | | `s3.resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `t1.small` | +| `lifecycle` | S3 bucket lifecycle configuration. | `object` | `{}` | +| `lifecycle.enabled` | Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new. | `bool` | `false` | +| `lifecycle.storageClass` | StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again. | `string` | `""` | diff --git a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag index 49f252950..bf4084277 100644 --- a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag +++ b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag @@ -1 +1 @@ -ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 +ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 diff --git a/packages/extra/seaweedfs/templates/admin-credentials.yaml b/packages/extra/seaweedfs/templates/admin-credentials.yaml new file mode 100644 index 000000000..4b9f6dd33 --- /dev/null +++ b/packages/extra/seaweedfs/templates/admin-credentials.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.lifecycle.enabled (ne .Values.topology "Client") }} +{{- $name := printf "%s-admin-credentials" .Release.Name }} +{{- /* The admin reads these into its environment when it starts and nothing + restarts it on a Secret change, so a password regenerated on every render + would leave the Secret holding one the running server does not accept. + Keep the one already issued. */}} +{{- $password := randAlphaNum 40 | b64enc }} +{{- with lookup "v1" "Secret" .Release.Namespace $name }} +{{- with (dig "data" "adminPassword" "" .) }} +{{- $password = . }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} +type: Opaque +data: + adminUser: {{ "admin" | b64enc }} + adminPassword: {{ $password }} +{{- end }} diff --git a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml index dfd408dad..eb038fe8f 100644 --- a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml +++ b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml @@ -27,6 +27,10 @@ rules: - {{ $.Release.Name }}-filer - {{ $.Release.Name }}-db - {{ $.Release.Name }}-s3 + {{- if .Values.lifecycle.enabled }} + - {{ $.Release.Name }}-admin + - {{ $.Release.Name }}-worker + {{- end }} {{- if eq .Values.topology "Simple" }} - {{ $.Release.Name }}-volume {{- range $poolName, $pool := .Values.volume.pools }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs-db.yaml b/packages/extra/seaweedfs/templates/seaweedfs-db.yaml index c86677c4d..261fcc93e 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs-db.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs-db.yaml @@ -42,6 +42,6 @@ spec: resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.db.resourcesPreset .Values.db.resources $) | nindent 8 }} size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 27e5557e0..92b40ef4d 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -214,10 +214,21 @@ spec: namespace: cozy-system interval: 30s timeout: 10m + {{- /* Flux remediates a failed install by uninstalling and a failed upgrade + by rolling back, and with retries: -1 it does so on every attempt. A + release waiting on tenant quota for the admin volume would then roll + the s3 pods back and forth with the [grpc.s3] restart annotation. + RetryOnFailure keeps what was applied and retries it. */}} install: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 upgrade: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 valuesFrom: @@ -249,7 +260,7 @@ spec: type: "persistentVolumeClaim" size: "{{ .Values.volume.size }}" {{- with .Values.volume.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 {{- if .Values.volume.diskType }} @@ -268,7 +279,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $pool.size | default $.Values.volume.size }}" {{- with ($pool.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 extraArgs: @@ -284,7 +295,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $zone.size | default $.Values.volume.size }}" {{- with ($zone.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 nodeSelector: | @@ -309,7 +320,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $pool.size | default $zone.size | default $.Values.volume.size }}" {{- with ($pool.storageClass | default $zone.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 nodeSelector: | @@ -338,6 +349,7 @@ spec: s3: replicas: {{ .Values.s3.replicas }} ingress: + enabled: {{ not (empty $ingress) }} className: {{ $ingress }} host: {{ .Values.host | default (printf "s3.%s" $host) }} annotations: @@ -370,6 +382,16 @@ spec: secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} + admin: + enabled: {{ .Values.lifecycle.enabled }} + secret: + existingSecret: {{ .Release.Name }}-admin-credentials + {{- with .Values.lifecycle.storageClass }} + data: + storageClass: {{ . | quote }} + {{- end }} + worker: + enabled: {{ .Values.lifecycle.enabled }} cosi: driverName: "{{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io" bucketClassName: "{{ .Release.Namespace }}" @@ -502,4 +524,22 @@ spec: app.kubernetes.io/component: s3 app.kubernetes.io/name: seaweedfs version: {{ $.Chart.Version }} +{{- if .Values.lifecycle.enabled }} +{{- range $component := list "admin" "worker" }} +--- +apiVersion: cozystack.io/v1alpha1 +kind: WorkloadMonitor +metadata: + name: {{ $.Release.Name }}-{{ $component }} +spec: + replicas: 1 + minReplicas: 1 + kind: seaweedfs + type: {{ $component }} + selector: + app.kubernetes.io/component: {{ $component }} + app.kubernetes.io/name: seaweedfs + version: {{ $.Chart.Version }} +{{- end }} +{{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/tests/db_storage_class_test.yaml b/packages/extra/seaweedfs/tests/db_storage_class_test.yaml new file mode 100644 index 000000000..da4f6cd2d --- /dev/null +++ b/packages/extra/seaweedfs/tests/db_storage_class_test.yaml @@ -0,0 +1,30 @@ +suite: seaweedfs metadata database storage class reaches its release as a string + +templates: + - templates/seaweedfs-db.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + # A StorageClass name may be all digits; handed on unquoted it reaches the + # database release as a number, and a line break in it would open new keys + # in that release's values. + - it: hands a numeric storage class name on as a string + set: + db: + storageClass: "123" + asserts: + - equal: + path: spec.values.db.storageClass + value: "123" + + - it: keeps a storage class with a line break inside its own field + set: + db: + storageClass: "local\n replicas: 9" + asserts: + - equal: + path: spec.values.db.storageClass + value: "local\n replicas: 9" diff --git a/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml b/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml new file mode 100644 index 000000000..55acf8198 --- /dev/null +++ b/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml @@ -0,0 +1,90 @@ +suite: seaweedfs waits for ingress-nginx only in a namespace that runs it +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: + topology: Simple + replicationFactor: 2 + _cluster: + issuer-name: letsencrypt-prod + +tests: + - it: depends on ingress-nginx-system when the namespace runs its own ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: tenant-root + asserts: + - contains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-root + + - it: does not depend on ingress-nginx-system when the root tenant has no ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: "" + asserts: + - notContains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-root + + - it: does not depend on ingress-nginx-system when ingress lives in a parent namespace + release: + namespace: tenant-acme + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: acme.example.org + ingress: tenant-root + asserts: + - notContains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-acme + + - it: disables the S3 Ingress when the namespace has no ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: "" + asserts: + - equal: + path: spec.values.seaweedfs.s3.ingress.enabled + value: false + + - it: keeps the S3 Ingress when the namespace has an ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: tenant-root + asserts: + - equal: + path: spec.values.seaweedfs.s3.ingress.enabled + value: true + - equal: + path: spec.values.seaweedfs.s3.ingress.className + value: tenant-root diff --git a/packages/extra/seaweedfs/tests/lifecycle_test.yaml b/packages/extra/seaweedfs/tests/lifecycle_test.yaml new file mode 100644 index 000000000..633514280 --- /dev/null +++ b/packages/extra/seaweedfs/tests/lifecycle_test.yaml @@ -0,0 +1,330 @@ +suite: seaweedfs lifecycle switch reaches the data plane + +# lifecycle.enabled decides whether the data-plane release runs the admin +# server and the worker that enforce bucket lifecycle rules. Both flags are +# sent either way, so the data-plane chart's own default never decides. + +templates: + - templates/seaweedfs.yaml + - templates/admin-credentials.yaml + - templates/dashboard-resourcemap.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + # The upstream walker deletes without an object identity, so a key + # overwritten between its listing and the delete RPC goes too. Applying + # stored rules is therefore an operator's decision, not an upgrade's. + - it: keeps the lifecycle admin and worker off by default + template: templates/seaweedfs.yaml + set: &platform + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: false + - equal: + path: spec.values.seaweedfs.worker.enabled + value: false + + - it: runs the lifecycle admin and worker when lifecycle is enabled + template: templates/seaweedfs.yaml + set: &enabled + <<: *platform + lifecycle: + enabled: true + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: true + - equal: + path: spec.values.seaweedfs.worker.enabled + value: true + - equal: + path: spec.values.seaweedfs.admin.secret.existingSecret + value: seaweedfs-admin-credentials + + - it: turns both off when lifecycle is disabled + template: templates/seaweedfs.yaml + set: + <<: *platform + lifecycle: + enabled: false + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: false + - equal: + path: spec.values.seaweedfs.worker.enabled + value: false + + # volume.storageClass is chosen for volume data, and in MultiZone it can be + # a class local to one zone, while the admin is not pinned to any zone. Its + # volume therefore takes its own class, and the cluster default otherwise. + - it: puts the admin volume on the lifecycle storage class + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: replicated + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.data.storageClass + value: replicated + + - it: does not put the admin volume on the volume storage class + template: templates/seaweedfs.yaml + set: + <<: *enabled + topology: MultiZone + replicationFactor: 1 + volume: + storageClass: zone-a-local + zones: + a: {} + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - notExists: + path: spec.values.seaweedfs.admin.data + + - it: leaves the admin volume on the cluster default when lifecycle names none + template: templates/seaweedfs.yaml + set: *enabled + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - notExists: + path: spec.values.seaweedfs.admin.data + + # A StorageClass name may be all digits. Handed on unquoted it reaches the + # data-plane release as a number, and the claim it ends up in is rejected. + - it: hands a numeric storage class name on as a string + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "123" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.data.storageClass + value: "123" + + - it: refuses a storage class that is not a StorageClass name + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "replicated\n image:\n repository: example/other" + asserts: + - failedTemplate: + errorPattern: "/lifecycle/storageClass" + + - it: refuses a storage class longer than a StorageClass name can be + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - failedTemplate: + errorPattern: "/lifecycle/storageClass" + + - it: monitors the admin and the worker while lifecycle is on + template: templates/seaweedfs.yaml + set: *enabled + asserts: + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-admin + any: true + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-worker + any: true + - equal: + path: spec.selector + value: + app.kubernetes.io/component: worker + app.kubernetes.io/name: seaweedfs + documentSelector: + path: metadata.name + value: seaweedfs-worker + - equal: + path: spec.selector + value: + app.kubernetes.io/component: admin + app.kubernetes.io/name: seaweedfs + documentSelector: + path: metadata.name + value: seaweedfs-admin + + - it: does not monitor them when lifecycle is off + template: templates/seaweedfs.yaml + set: + <<: *platform + lifecycle: + enabled: false + asserts: + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-admin + any: true + not: true + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-worker + any: true + not: true + + - it: generates admin credentials for a new instance + template: templates/admin-credentials.yaml + set: *enabled + asserts: + - isKind: + of: Secret + - equal: + path: metadata.name + value: seaweedfs-admin-credentials + - equal: + path: data.adminUser + value: YWRtaW4= + - matchRegex: + path: data.adminPassword + pattern: '^.{40,}$' + + # A new password on every render would leave the running admin on the old + # one, since it reads the Secret into its environment once at start. + - it: keeps the password of an existing Secret + template: templates/admin-credentials.yaml + set: *enabled + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: seaweedfs-admin-credentials + namespace: tenant-root + data: + adminUser: YWRtaW4= + adminPassword: a2VwdA== + asserts: + - equal: + path: data.adminPassword + value: a2VwdA== + + - it: renders no credentials when lifecycle is off + template: templates/admin-credentials.yaml + set: + <<: *platform + lifecycle: + enabled: false + asserts: + - hasDocuments: + count: 0 + + # A Client instance runs no data plane of its own; its rules are enforced + # where the master it points at runs. + - it: renders no credentials for a Client instance + template: templates/admin-credentials.yaml + set: + <<: *enabled + topology: Client + filer: + grpcHost: seaweedfs-filer.tenant-other.svc + asserts: + - hasDocuments: + count: 0 + + # The admin PVC and the two lifecycle pods can leave an upgrade waiting on a + # full tenant quota. Under Flux's default rollback remediation each failed + # attempt would roll back the [grpc.s3] section and its restart annotation + # and the next one would bring them back, restarting the s3 pods on every + # cycle. Retrying in place keeps what was applied while the release waits. + - it: retries the data-plane release in place instead of rolling it back + template: templates/seaweedfs.yaml + set: *platform + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.install.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.remediation.retries + value: -1 + + # The dashboard reads each WorkloadMonitor by name through this Role, so a + # monitor missing from it never shows up for the tenant. + - it: lets the dashboard read the admin and worker monitors + template: templates/dashboard-resourcemap.yaml + set: *enabled + documentSelector: + path: kind + value: Role + asserts: + - contains: + path: rules[2].resourceNames + content: seaweedfs-admin + - contains: + path: rules[2].resourceNames + content: seaweedfs-worker + + - it: grants no lifecycle monitors when lifecycle is off + template: templates/dashboard-resourcemap.yaml + set: + <<: *platform + lifecycle: + enabled: false + documentSelector: + path: kind + value: Role + asserts: + - notContains: + path: rules[2].resourceNames + content: seaweedfs-admin + - notContains: + path: rules[2].resourceNames + content: seaweedfs-worker diff --git a/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml b/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml new file mode 100644 index 000000000..2474f3358 --- /dev/null +++ b/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml @@ -0,0 +1,102 @@ +suite: seaweedfs volume storage class reaches the data plane as a string + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: &platform + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: hands a numeric storage class name on as a string + set: + <<: *platform + volume: + storageClass: "123" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volume.dataDirs[0].storageClass + value: "123" + + # The value is written into the values of another release; a line break in + # it must not open new keys there. + - it: keeps a storage class with a line break inside its own field + set: + <<: *platform + volume: + storageClass: "local\n image: example/other" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volume.dataDirs[0].storageClass + value: "local\n image: example/other" + - notExists: + path: spec.values.seaweedfs.volume.dataDirs[0].image + + # A pool or zone class is quoted by hand around the value, which a quote + # character inside it closes early, breaking the release values. + - it: keeps a pool storage class with a quote inside its own field + set: + <<: *platform + volume: + pools: + fast: + diskType: ssd + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.fast.dataDirs[0].storageClass + value: 'local"x' + + - it: keeps a zone storage class with a quote inside its own field + set: + <<: *platform + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.dataDirs[0].storageClass + value: 'local"x' + + - it: keeps a zone pool storage class with a quote inside its own field + set: + <<: *platform + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + pools: + fast: + diskType: ssd + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a-fast.dataDirs[0].storageClass + value: 'local"x' diff --git a/packages/extra/seaweedfs/values.schema.json b/packages/extra/seaweedfs/values.schema.json index ab9b953ac..0df1bef8f 100644 --- a/packages/extra/seaweedfs/values.schema.json +++ b/packages/extra/seaweedfs/values.schema.json @@ -868,6 +868,25 @@ ] } } + }, + "lifecycle": { + "description": "S3 bucket lifecycle configuration.", + "type": "object", + "default": {}, + "properties": { + "enabled": { + "description": "Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new.", + "type": "boolean", + "default": false + }, + "storageClass": { + "description": "StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again.", + "type": "string", + "default": "", + "maxLength": 253, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$" + } + } } } } diff --git a/packages/extra/seaweedfs/values.yaml b/packages/extra/seaweedfs/values.yaml index 0aa569bc4..fdc58a11e 100644 --- a/packages/extra/seaweedfs/values.yaml +++ b/packages/extra/seaweedfs/values.yaml @@ -192,3 +192,21 @@ s3: replicas: 2 resources: {} resourcesPreset: "t1.small" + +## @typedef {struct} Lifecycle - Enforcement of S3 bucket lifecycle rules. +## @field {bool} [enabled] - Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new. +## @field {string} [storageClass] - StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again. +## @maxLength 253 +## @pattern ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$ + +## @param {Lifecycle} [lifecycle] - S3 bucket lifecycle configuration. +lifecycle: + # Off by default. The daily walk of SeaweedFS (4.45 through 4.52) sends its + # expiry deletes without an object identity, which the s3 server takes as + # "skip the check" (weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go, + # identityMatches in weed/s3api/s3api_internal_lifecycle.go), and it does not + # re-check the object's age. A key overwritten between the walk's listing and + # the delete loses its new content, which a tenant bucket that rewrites a + # fixed key can hit. On by default only once that path carries identity. + enabled: false + storageClass: "" diff --git a/packages/system/backupstrategy-controller/Makefile b/packages/system/backupstrategy-controller/Makefile index 94b1931d9..aabc11d17 100644 --- a/packages/system/backupstrategy-controller/Makefile +++ b/packages/system/backupstrategy-controller/Makefile @@ -6,9 +6,9 @@ include ../../../hack/package.mk # PHONY so a stray file/dir named `image` cannot mark the aggregate up to date # and skip both builds. -.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client +.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup -image: image-backupstrategy-controller image-redis-backup image-backup-http-client +image: image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup image-backupstrategy-controller: docker buildx build -f images/backupstrategy-controller/Dockerfile ../../.. \ @@ -46,5 +46,16 @@ image-backup-http-client: image-redis-backup yq -i '.backupStrategyController.chBackupClientImage = strenv(IMAGE) | .backupStrategyController.rabbitmqBackupClientImage = strenv(IMAGE)' values.yaml rm -f images/backup-http-client.json +# Chained after image-backup-http-client: one `yq -i` writer at a time. +image-kubernetes-backup: image-backup-http-client + docker buildx build -f images/kubernetes-backup/Dockerfile images/kubernetes-backup \ + $(call image-tags,kubernetes-backup,v$(COZYSTACK_VERSION)) \ + $(call cache-args,kubernetes-backup) \ + --metadata-file images/kubernetes-backup.json \ + $(BUILDX_ARGS) + IMAGE="$(REGISTRY)/kubernetes-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubernetes-backup.json -o json -r)" \ + yq -i '.backupStrategyController.kubernetesBackupClientImage = strenv(IMAGE)' values.yaml + rm -f images/kubernetes-backup.json + test: helm unittest . diff --git a/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml new file mode 100644 index 000000000..0b681c5be --- /dev/null +++ b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml @@ -0,0 +1,147 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: kubernetes.strategy.backups.cozystack.io +spec: + group: strategy.backups.cozystack.io + names: + kind: Kubernetes + listKind: KubernetesList + plural: kubernetes + singular: kubernetes + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Kubernetes defines a backup strategy for a tenant Kubernetes application. + A tenant cluster keeps its state in two places the platform can reach: the + cluster's key prefix in the tenant's shared etcd DataStore, and host-side + objects (control-plane PKI Secrets, the application HelmReleases, and the + DataVolumes that back the tenant's PersistentVolumes). The strategy captures + the first with a one-shot Job and the second with Velero. The two are taken + one after another, so a backup is crash-consistent at best. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KubernetesSpec specifies the desired behavior of a Kubernetes + backup strategy. + properties: + certificateWarningThreshold: + description: |- + CertificateWarningThreshold is how close to expiry a control-plane + certificate may be before a backup records a warning. Defaults to 720h. + type: string + template: + description: |- + Template holds a PodTemplateSpec for the Job that dumps the cluster's + key prefix from its etcd DataStore to object storage, writes it back on + restore, and deletes the object on cleanup. The Job runs in the + application namespace. Helm-like Go templates are supported in every + string field and are rendered against: + - `.Release.Name` and `.Release.Namespace` — the application's name and + namespace; `.Release.Prefix` is `kubernetes-`. + - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + - `.ObjectKey` — the object-storage key of this backup's dump. + - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + - `.Datastore.Endpoints` — comma-separated etcd client URLs. + - `.Datastore.CertificateSecret` — the Secret holding the cluster's + etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + - `.Parameters` — the resolved BackupClass strategy parameters. + + Stored schemaless for the same CRD-size reason as redis_types.go. + x-kubernetes-preserve-unknown-fields: true + velero: + description: |- + Velero holds the Velero Backup and Restore specs for the host-side + objects and volumes. Rendered against `.Application` and `.Parameters`, + like the Velero strategy. + x-kubernetes-preserve-unknown-fields: true + required: + - template + - velero + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile new file mode 100644 index 000000000..b897e51be --- /dev/null +++ b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile @@ -0,0 +1,14 @@ +# kubernetes-backup runs the Kubernetes strategy Pod: it dumps a tenant +# cluster's key prefix from its etcd DataStore with etcdctl, moves the dump to +# S3 with curl --aws-sigv4, and writes it back with etcdctl on restore. etcdctl +# matches the etcd the platform deploys. Everything is installed at build time +# so the Pods never run `apk add` and work on air-gapped clusters. +FROM quay.io/coreos/etcd:v3.6.11@sha256:6ae247c7666ceec554c51ba1f9bc8dd2212f975370dbd65710c9ca0e36ae1fff AS etcd + +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 + +RUN apk add --no-cache curl jq ca-certificates coreutils + +COPY --from=etcd /usr/local/bin/etcdctl /usr/local/bin/etcdctl + +USER 65532:65532 diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index f64c77b88..b481d1e1c 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -190,3 +190,28 @@ true {{- end -}} {{- end -}} {{- end -}} + +{{/* + backupStorage.forcePathStyle as "true", "false", or "" when unset. Its + consumers read it differently: the controller and Velero parse it with + strconv.ParseBool, the etcd and MongoDB strategies write it into a boolean + field that Helm decodes as YAML 1.1 (so "yes" or "off" is a boolean there), + and the Kafka and RabbitMQ scripts compare the string with "false". + Normalising it here means a spelling cannot mean path style to some drivers + and not to others. The spellings accepted are those of strconv.ParseBool and + of YAML 1.1 booleans, a superset of the boolean spellings the etcd and + MongoDB strategies accepted; any other value fails the render. +*/}} +{{- define "backupstrategy-controller.forcePathStyle" -}} +{{- $value := .Values.backupStorage.forcePathStyle -}} +{{- if not (kindIs "invalid" $value) -}} +{{- $raw := toString $value -}} +{{- if has $raw (list "1" "t" "T" "true" "True" "TRUE" "y" "Y" "yes" "Yes" "YES" "on" "On" "ON") -}} +true +{{- else if has $raw (list "0" "f" "F" "false" "False" "FALSE" "n" "N" "no" "No" "NO" "off" "Off" "OFF") -}} +false +{{- else if ne $raw "" -}} +{{- fail (printf "backupStorage.forcePathStyle must be a boolean or unset, got %q" $raw) -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml index 3f3b271e3..347c704d1 100644 --- a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml +++ b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml @@ -78,6 +78,13 @@ spec: apiGroup: strategy.backups.cozystack.io kind: Redis name: cozy-default-redis + - application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes - application: apiGroup: apps.cozystack.io kind: Kafka diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index 887b7e64e..2fc9b9ee6 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -47,7 +47,7 @@ spec: - name: BACKUP_STORAGE_REGION value: {{ .Values.backupStorage.region | quote }} - name: BACKUP_STORAGE_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} - name: BACKUP_STORAGE_SYSTEM_NAMESPACES value: {{ .Values.backupStorage.systemNamespaces | join "," | quote }} {{- if .Values.backupStorage.reconcileDefaultObjects }} diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..735e16308 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -62,7 +62,25 @@ rules: # through the dynamic client, so no list/watch) - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] +# Kubernetes strategy: read where a tenant cluster keeps its etcd prefix, and +# stop and resume its control plane, worker pools and cluster-autoscaler around +# an in-place restore. +- apiGroups: ["kamaji.clastix.io"] + resources: ["tenantcontrolplanes", "datastores"] + verbs: ["get"] +- apiGroups: ["controlplane.cluster.x-k8s.io"] + resources: ["kamajicontrolplanes"] + verbs: ["get", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machinedeployments"] + verbs: ["get", "list", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machines"] + verbs: ["list", "patch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "patch"] # HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR # and deletes old. patch additionally lets the DefaultObjectsGate stamp # reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 21d05f8a4..3e6747966 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -58,32 +58,75 @@ spec: # text/template's missing-key marker, when a controller that # predates archive names renders this template during an upgrade. archive="{{ printf `{{ .ArchiveName | default "" }}` }}" - for _ in $(seq 1 60); do - if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi - sleep 5 - done + # Every request but the POST is retried on failure, every 5s, + # for $2 attempts in all: once the operation is accepted, a + # dropped poll must not fail the attempt, since the Job retry it + # causes cannot tell a lost poll from a lost operation. + api_get() { + n=0 + while true; do + if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then + printf '%s\n' "${out}" + return 0 + fi + n=$((n + 1)) + if [ "${n}" -ge "$2" ]; then + echo "GET ${api}$1 failed ${n} times in a row" >&2 + return 1 + fi + sleep 5 + done + } + # The sidecar answers 423 to a new operation while any other is + # in progress, so starting before it is idle burns the attempt. + # The exception is the operation $1 itself already running, as + # left by an earlier attempt of this Job that died: it is taken + # over rather than started twice. + wait_turn() { + for _ in $(seq 1 60); do + if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then + turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" ' + if any(.[]; .status == "in progress" and .command == $cmd) then "join" + elif any(.[]; .status == "in progress") then "busy" + else "idle" end') || turn= + case "${turn}" in join|idle) return 0 ;; esac + fi + sleep 5 + done + echo "clickhouse-backup sidecar at ${api} did not become idle" >&2 + exit 1 + } if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" - # Only that older controller leaves the name empty, and it - # records no name on the Backup either. + # Only a controller that predates archive names leaves it + # empty, and that one records no name on the Backup either. target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}" - # A retried Pod gets the same name. Wait on an attempt the - # sidecar is still running, which would answer a new one with - # 423, and ask for resume, so a local or partial remote backup - # an earlier attempt left is resumed rather than refused as - # existing, whatever use_resumable_state the sidecar runs with. - state=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last | .status // ""') - if [ "${state}" != "in progress" ]; then - echo "create_remote ${target}" + # A retried Pod gets the same name, so it takes over an attempt + # the sidecar is still running, and otherwise asks for resume, + # so a local or partial remote backup an earlier attempt left is + # resumed rather than refused as existing, whatever + # use_resumable_state the sidecar runs with. The sidecar logs + # the request as "create_remote --resume ". The name is + # this BackupJob's alone, so an upload of it that last ended in + # success is an earlier attempt's finished result: resuming + # over it would upload the archive again and leave it broken if + # that upload died halfway. + cmd="create_remote --resume ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then + echo "${cmd} already succeeded in an earlier attempt" + else + echo "${cmd}" curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null fi else app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}" backup="{{ printf "{{ .Backup.Name }}" }}" prefix="clickhouse-${app}-" - remote=$(curl -fsS ${auth} "${api}/backup/list/remote") + remote=$(api_get /backup/list/remote 60) if [ -n "${archive}" ]; then if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2 @@ -104,19 +147,39 @@ spec: exit 1 fi fi - echo "restore_remote ${target}" - curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + # restore_remote drops every table it restores before + # recreating it, so a second run discards whatever was written + # after the first one finished. Only a running one is taken + # over: the action log is never trimmed, and a finished row of + # this archive may be left by an earlier RestoreJob. + cmd="restore_remote ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + else + echo "${cmd}" + curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + fi fi while true; do - row=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last') + actions=$(api_get /backup/actions 6) + row=$(printf '%s\n' "${actions}" \ + | jq -rs --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last') + # The sidecar logs the operation before acknowledging the POST + # and keeps that log only in memory, so a missing row means it + # restarted and the operation is gone; nothing will resume it. + if [ "${row}" = "null" ]; then + echo "operation ${target} is no longer in the sidecar action log; the sidecar restarted and dropped it" >&2 + exit 1 + fi status=$(echo "${row}" | jq -r '.status // ""') case "${status}" in success) echo "operation succeeded" ; exit 0 ;; - error) + "in progress") ;; + *) err=$(echo "${row}" | jq -r '.error // "unknown"') - echo "operation failed: ${err}" >&2 + echo "operation ended with status ${status}: ${err}" >&2 exit 1 ;; esac diff --git a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml index 5183af45b..e7b2afcad 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml @@ -15,7 +15,7 @@ spec: endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} key: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/" | quote }} region: {{ .Values.backupStorage.region | quote }} - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} # The etcd-operator reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY # from this Secret by default (no explicit key fields like the # MariaDB strategy's *SecretKeyRef). The projector writes exactly diff --git a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml index 1f14465ed..fb015ba66 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml @@ -69,7 +69,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/usr/bin/bash", "-c"] # The driver script is kept as a standalone shell file and # embedded verbatim so hack/kafka-backup-script.bats can execute diff --git a/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml new file mode 100644 index 000000000..3fab192ad --- /dev/null +++ b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml @@ -0,0 +1,304 @@ +{{/* + Two template layers, as in strategy-redis-default.yaml: Helm renders this + file, then the strategy controller renders the resulting CR per BackupJob. + Literal "{{ .Foo }}" for the second layer goes through printf. + + Gated on both a Velero BSL and a resolved $bucketName: the Velero half of + the backup goes to the BSL and the etcd dump to the bucket. Also gated on + its CRD being established, for the upgrade of an existing install that + first ships this kind: see strategy-mongodb-default.yaml. +*/}} +{{- $bucketName := include "backupstrategy-controller.bucketName" . -}} +{{- $crdEstablished := include "backupstrategy-controller.crdEstablished" (dict "root" . "crd" "kubernetes.strategy.backups.cozystack.io") -}} +{{- if and .Values.velero.bslEnabled $bucketName $crdEstablished -}} +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Kubernetes +metadata: + name: cozy-default-kubernetes + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} +spec: + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: {{ int .Values.backupStrategyController.kubernetesBackupActiveDeadlineSeconds }} + volumes: + {{- /* + The cluster's own etcd client certificate, the one its kube-apiserver + uses. Optional so a missing Secret fails the script with a clear cause + instead of wedging the Pod on FailedMount. + */}} + - name: etcd-client + secret: + secretName: {{ printf "{{ .Datastore.CertificateSecret }}" | quote }} + optional: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + secret: + secretName: {{ .Values.backupStorage.endpointCASecretName | quote }} + optional: true + {{- end }} + containers: + - name: kubernetes-backup + image: "{{ .Values.backupStrategyController.kubernetesBackupClientImage }}" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etcd-client + mountPath: /etc/etcd-client + readOnly: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + mountPath: /etc/s3-ca + readOnly: true + {{- end }} + env: + - name: ETCDCTL_ENDPOINTS + value: {{ printf "{{ .Datastore.Endpoints }}" | quote }} + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: endpoint + optional: true + - name: S3_BUCKET + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: bucketName + optional: true + - name: S3_REGION + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: region + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: S3_INSECURE_SKIP_VERIFY + value: {{ .Values.backupStorage.insecureSkipTLSVerify | quote }} + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + : "${S3_ENDPOINT:?missing endpoint in cozy-backups-creds}" + : "${S3_BUCKET:?missing bucketName in cozy-backups-creds}" + : "${S3_REGION:?missing region in cozy-backups-creds}" + : "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in cozy-backups-creds}" + : "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in cozy-backups-creds}" + + MODE="{{ printf "{{ .Mode }}" }}" + KEY="{{ printf "{{ .ObjectKey }}" }}" + SCHEMA="{{ printf "{{ .Datastore.Prefix }}" }}" + SOURCE_SCHEMA="{{ printf "{{ .Datastore.SourcePrefix }}" }}" + + case "${S3_ENDPOINT}" in + http://*|https://*) BASE="${S3_ENDPOINT}" ;; + *) BASE="https://${S3_ENDPOINT}" ;; + esac + TLS="" + case "${BASE}" in + https://*) + if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt"; + elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt"; + elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k"; + fi + ;; + esac + s3() { + curl -fsS ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@" + } + OBJ="${BASE}/${S3_BUCKET}/${KEY}" + {{- /* + Where a restore keeps the prefix it replaces. One per backup, so + cleanup removes it with the dump. A restore writes it only when it + is absent and deletes it on success, so after a failed restore it + holds the prefix from before the first failed attempt and a retry + cannot overwrite it with a partial one. + */}} + PRE_KEY="${KEY}.pre-restore" + PRE_OBJ="${BASE}/${S3_BUCKET}/${PRE_KEY}" + + if [ "${MODE}" = "cleanup" ]; then + for o in "${OBJ}" "${PRE_OBJ}"; do + echo "deleting ${o#"${BASE}/"}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${o}") + case "${code}" in + 2??|404) echo "deleted (HTTP ${code})" ;; + *) echo "delete failed: HTTP ${code}" >&2; exit 1 ;; + esac + done + exit 0 + fi + + if [ "${MODE}" = "verify" ]; then + s3 -I "${OBJ}" > /dev/null + echo "s3://${S3_BUCKET}/${KEY} exists" + exit 0 + fi + + {{- /* + The prefix comes from the TenantControlPlane: Kamaji's schema is the + TenantControlPlane's UID, or _ for one created by + an older Kamaji. Refuse anything else: an empty or slash-bearing + value would turn the restore's `del --prefix` into a wipe of every + cluster in the shared etcd. + */}} + for s in "${SCHEMA}" "${SOURCE_SCHEMA}"; do + case "${s}" in + ""|*[!a-z0-9_-]*) echo "refusing unexpected datastore prefix '${s}'" >&2; exit 1 ;; + esac + done + PREFIX="/${SCHEMA}/" + {{- /* + The prefix the dump was taken under. A rebuilt cluster has a new + TenantControlPlane and so a new schema; its keys are written under + the new prefix. + */}} + SOURCE_PREFIX="/${SOURCE_SCHEMA}/" + for f in ca.crt server.crt server.key; do + [ -f "/etc/etcd-client/${f}" ] || { echo "etcd client certificate is missing /etc/etcd-client/${f}" >&2; exit 1; } + done + export ETCDCTL_CACERT=/etc/etcd-client/ca.crt + export ETCDCTL_CERT=/etc/etcd-client/server.crt + export ETCDCTL_KEY=/etc/etcd-client/server.key + + if [ "${MODE}" = "backup" ]; then + etcdctl get "${PREFIX}" --prefix -w json > /tmp/dump.json + count=$(jq '.kvs // [] | length' /tmp/dump.json) + if [ "${count}" -eq 0 ]; then + echo "etcd prefix ${PREFIX} is empty" >&2 + exit 1 + fi + echo "dumped ${count} keys under ${PREFIX}; uploading s3://${S3_BUCKET}/${KEY}" + s3 -X PUT --upload-file /tmp/dump.json "${OBJ}" + elif [ "${MODE}" = "restore" ]; then + {{- /* + Everything that can fail without touching etcd runs before the + prefix is deleted: the download, the check that every key belongs + to this cluster, and a copy of the prefix as it is now, which is + what an operator restores from if a failure comes after the delete. + */}} + echo "downloading s3://${S3_BUCKET}/${KEY}" + s3 -o /tmp/dump.json "${OBJ}" + jq -e '.kvs | length > 0' /tmp/dump.json >/dev/null || { echo "dump holds no keys" >&2; exit 1; } + jq -e --arg p "${SOURCE_PREFIX}" 'all(.kvs[]; .key | @base64d | startswith($p))' /tmp/dump.json >/dev/null \ + || { echo "dump holds keys outside ${SOURCE_PREFIX}" >&2; exit 1; } + {{- /* + etcdctl put cannot write an empty value, so a dump holding one + must stop here rather than halfway through the write. + */}} + jq -e 'all(.kvs[]; (.value // "") != "")' /tmp/dump.json >/dev/null \ + || { echo "dump holds a key with an empty value, which etcdctl cannot write" >&2; exit 1; } + jq -r '.kvs[] | [.key, .value] | @tsv' /tmp/dump.json > /tmp/kvs.tsv + {{- /* + Only a definite answer decides about the copy: an existing one is + kept, a missing one is taken, and any other reply (a 5xx, a + timeout) stops the restore while the prefix is still intact, + rather than overwriting the copy an earlier failed restore left. + */}} + code=$(curl -sS -o /dev/null -w '%{http_code}' -I ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "${PRE_OBJ}") || code=000 + case "${code}" in + 200) echo "keeping s3://${S3_BUCKET}/${PRE_KEY} from an earlier failed restore" ;; + 404) + etcdctl get "${PREFIX}" --prefix -w json > /tmp/before.json + echo "saving the current prefix to s3://${S3_BUCKET}/${PRE_KEY}" + s3 -X PUT --upload-file /tmp/before.json "${PRE_OBJ}" + ;; + *) echo "cannot tell whether s3://${S3_BUCKET}/${PRE_KEY} exists: HTTP ${code}" >&2; exit 1 ;; + esac + {{- /* + Delete first so keys created after the backup do not survive the + restore. Events and master leases are bound to etcd leases that + did not survive; written back without one they would never expire, + so they are skipped. They are matched as whole top-level resources + under the prefix, not as a path segment anywhere, which would also + drop every object in a namespace named "events". + */}} + {{- /* + The termination message tells the driver that this Pod got as far + as changing the prefix: only then is the control plane left + stopped after a failure. + */}} + echo writing > /dev/termination-log + etcdctl del "${PREFIX}" --prefix + n=0 + while IFS="$(printf '\t')" read -r k v; do + key=$(printf '%s' "${k}" | base64 -d) + rel="${key#"${SOURCE_PREFIX}"}" + case "${rel}" in events/*|masterleases/*) continue ;; esac + printf '%s' "${v}" | base64 -d > /tmp/value + etcdctl put "${PREFIX}${rel}" < /tmp/value > /dev/null + n=$((n + 1)) + done < /tmp/kvs.tsv + echo "restored ${n} keys under ${PREFIX}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${PRE_OBJ}") || code=000 + case "${code}" in + 2??|404) echo "dropped s3://${S3_BUCKET}/${PRE_KEY}" ;; + *) echo "WARNING: could not drop s3://${S3_BUCKET}/${PRE_KEY} (HTTP ${code}); a later restore would keep it as the pre-restore copy, so delete it by hand" >&2 ;; + esac + else + echo "unknown mode: ${MODE}" >&2 + exit 1 + fi + resources: + {{- toYaml .Values.backupStrategyController.kubernetesBackupResources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + velero: + {{- /* + Secrets (control-plane and Talos PKI, by the Kamaji and Helm instance + labels) and the DataVolumes and PVCs backing the cluster's + PersistentVolumes (the CSI driver stamps the cluster-name label on them; + worker system disks do not carry it). The HelmReleases are recorded by the + driver on the Backup instead: selecting them by application label would + also select worker disks the lineage webhook labels the same way. The + driver sets the restore's resources and selectors per phase. + */}} + spec: + includedNamespaces: + - {{ printf "{{ .Application.metadata.namespace }}" | quote }} + orLabelSelectors: + - matchLabels: + kamaji.clastix.io/name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + app.kubernetes.io/instance: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + cluster.x-k8s.io/cluster-name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + includedResources: + - secrets + - persistentvolumeclaims + - datavolumes.cdi.kubevirt.io + includeClusterResources: false + storageLocation: cozy-default + snapshotVolumes: true + snapshotMoveData: true + ttl: 720h0m0s + itemOperationTimeout: 24h0m0s + restoreSpec: + includeClusterResources: false +{{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml index 1bddbde16..3bff9472b 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml @@ -76,6 +76,6 @@ spec: prefix: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}" | quote }} region: {{ .Values.backupStorage.region | quote }} credentialsSecret: cozy-backups-creds - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} insecureSkipTLSVerify: {{ .Values.backupStorage.insecureSkipTLSVerify }} {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml index 9515945d2..28c9250f8 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml @@ -98,7 +98,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/bin/sh", "-c"] args: - | diff --git a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml index e50f3a1fc..a01f480ac 100644 --- a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml +++ b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml @@ -31,7 +31,7 @@ spec: key: cloud config: region: {{ .Values.backupStorage.region | quote }} - s3ForcePathStyle: {{ .Values.backupStorage.forcePathStyle | quote }} + s3ForcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} s3Url: {{ include "backupstrategy-controller.endpoint" . | quote }} {{- end }} {{- end }} diff --git a/packages/system/backupstrategy-controller/tests/deployment_test.yaml b/packages/system/backupstrategy-controller/tests/deployment_test.yaml index 6c17f5f58..f92455a8d 100644 --- a/packages/system/backupstrategy-controller/tests/deployment_test.yaml +++ b/packages/system/backupstrategy-controller/tests/deployment_test.yaml @@ -54,3 +54,12 @@ tests: name: BACKUP_STORAGE_SECRET_NAMESPACE value: tenant-root any: true + + # A value no consumer took as a boolean is refused once, at render time; see + # tests/force_path_style_test.yaml for what each consumer receives. + - it: "forcePathStyle that is no boolean spelling fails the render and names the value" + set: + backupStorage.forcePathStyle: "maybe" + asserts: + - failedTemplate: + errorPattern: 'backupStorage\.forcePathStyle must be a boolean or unset, got "maybe"' diff --git a/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml new file mode 100644 index 000000000..eda742951 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml @@ -0,0 +1,501 @@ +suite: forcePathStyle — one normalised boolean for every consumer + +# backupStorage.forcePathStyle reaches drivers that read it differently: the +# controller and Velero parse it with strconv.ParseBool, the etcd and MongoDB +# strategies write it into a boolean field Helm decodes as YAML 1.1, and the +# Kafka and RabbitMQ scripts compare the string with "false". The chart accepts +# the spellings of both strconv.ParseBool and YAML 1.1, hands each consumer +# "true" or "false", and refuses anything else, so a spelling cannot mean path +# style to one driver and not to another. + +templates: + - templates/deployment.yaml + - templates/strategy-etcd-default.yaml + - templates/strategy-mongodb-default.yaml + - templates/strategy-kafka-default.yaml + - templates/strategy-rabbitmq-default.yaml + - templates/velero-bsl.yaml + +release: + name: t + namespace: cozy-backup-controller + +set: + backupStorage.bucketNameOverride: test-bucket + +tests: + - it: "forcePathStyle '1' reaches the controller as true" + set: + backupStorage.forcePathStyle: "1" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 't' reaches the controller as true" + set: + backupStorage.forcePathStyle: "t" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'T' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "T" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'true' reaches the controller as true" + set: + backupStorage.forcePathStyle: "true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'True' reaches the controller as true" + set: + backupStorage.forcePathStyle: "True" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'TRUE' reaches the controller as true" + set: + backupStorage.forcePathStyle: "TRUE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'Y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'yes' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Yes' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'YES' reaches the controller as true" + set: + backupStorage.forcePathStyle: "YES" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'on' reaches the controller as true" + set: + backupStorage.forcePathStyle: "on" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'On' reaches the controller as true" + set: + backupStorage.forcePathStyle: "On" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'ON' reaches the controller as true" + set: + backupStorage.forcePathStyle: "ON" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle '0' reaches the controller as false" + set: + backupStorage.forcePathStyle: "0" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'f' reaches the controller as false" + set: + backupStorage.forcePathStyle: "f" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'F' reaches the controller as false" + set: + backupStorage.forcePathStyle: "F" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'false' reaches the controller as false" + set: + backupStorage.forcePathStyle: "false" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'False' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "False" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'FALSE' reaches the controller as false" + set: + backupStorage.forcePathStyle: "FALSE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'n' reaches the controller as false" + set: + backupStorage.forcePathStyle: "n" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'N' reaches the controller as false" + set: + backupStorage.forcePathStyle: "N" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'no' reaches the controller as false" + set: + backupStorage.forcePathStyle: "no" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'No' reaches the controller as false" + set: + backupStorage.forcePathStyle: "No" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'NO' reaches the controller as false" + set: + backupStorage.forcePathStyle: "NO" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'off' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Off' reaches the controller as false" + set: + backupStorage.forcePathStyle: "Off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'OFF' reaches the controller as false" + set: + backupStorage.forcePathStyle: "OFF" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle as a YAML boolean (the default) reaches every consumer as true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + - it: "forcePathStyle as a YAML boolean false reaches the scripts as false" + set: + backupStorage.forcePathStyle: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle as the number 1 reaches the boolean fields as true" + set: + backupStorage.forcePathStyle: 1 + asserts: + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + # Unset leaves each driver its own default, as before. + - it: "forcePathStyle null stays unset for every consumer" + set: + backupStorage.forcePathStyle: null + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: null + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle empty stays unset for every consumer" + set: + backupStorage.forcePathStyle: "" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml diff --git a/packages/system/backupstrategy-controller/tests/rendering_test.yaml b/packages/system/backupstrategy-controller/tests/rendering_test.yaml index 3db6adb14..941444070 100644 --- a/packages/system/backupstrategy-controller/tests/rendering_test.yaml +++ b/packages/system/backupstrategy-controller/tests/rendering_test.yaml @@ -22,6 +22,7 @@ templates: - templates/strategy-foundationdb-default.yaml - templates/strategy-velero-vminstance-default.yaml - templates/strategy-velero-vmdisk-default.yaml + - templates/strategy-kubernetes-default.yaml release: name: t @@ -88,11 +89,14 @@ tests: - hasDocuments: count: 0 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 0 + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 0 template: templates/velero-bsl.yaml - - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all eleven routes (FoundationDB intentionally unbound)" + - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all twelve routes (FoundationDB intentionally unbound)" asserts: - hasDocuments: count: 1 @@ -103,7 +107,18 @@ tests: template: templates/backupclass-default.yaml - lengthEqual: path: spec.strategies - count: 11 + count: 12 + template: templates/backupclass-default.yaml + - contains: + path: spec.strategies + content: + application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes template: templates/backupclass-default.yaml - contains: path: spec.strategies @@ -240,6 +255,18 @@ tests: - hasDocuments: count: 1 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 1 + template: templates/strategy-kubernetes-default.yaml + # The driver renders these per BackupJob; Helm must leave them literal. + - equal: + path: spec.template.spec.volumes[0].secret.secretName + value: "{{ .Datastore.CertificateSecret }}" + template: templates/strategy-kubernetes-default.yaml + - equal: + path: spec.velero.spec.orLabelSelectors[2].matchLabels["cluster.x-k8s.io/cluster-name"] + value: "kubernetes-{{ .Application.metadata.name }}" + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 1 template: templates/velero-bsl.yaml @@ -347,6 +374,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-foundationdb-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} # The ungated BackupClass keeps rendering, which is what makes the counts # above an assertion about the CRD gate rather than about a chart-wide # render failure. @@ -432,6 +460,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -449,6 +478,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} - it: "with every CRD Established and the BucketClaim resolved, every Strategy CR renders against the COSI bucket name" # The positive half of the case above, through the same live lookups and no @@ -471,6 +501,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -489,6 +520,7 @@ tests: - {hasDocuments: {count: 1}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 1}, template: templates/strategy-kubernetes-default.yaml} - it: "velero.bslEnabled=false: Velero strategies and BSL drop out while other strategies keep rendering" set: diff --git a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml index b13b88173..fa3406d58 100644 --- a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml +++ b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml @@ -13,11 +13,11 @@ release: namespace: cozy-backup-controller tests: - - it: "ClusterRole can read and delete DataVolumes" + - it: "ClusterRole can read, list and delete DataVolumes" asserts: - contains: path: rules content: apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 590d7aca2..8de863a27 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -37,6 +37,24 @@ backupStrategyController: cpu: "1" memory: 1Gi ephemeral-storage: 4Gi + # kubernetesBackupClientImage is rendered into the Kubernetes strategy Pod, + # which dumps a tenant cluster's etcd prefix with etcdctl and moves it to S3 + # with curl. Built from images/kubernetes-backup/; `make image` re-digests it. + kubernetesBackupClientImage: "ghcr.io/cozystack/cozystack/kubernetes-backup:v1.6.0" + # kubernetesBackupActiveDeadlineSeconds bounds one etcd dump/restore Job. + kubernetesBackupActiveDeadlineSeconds: 3600 + # kubernetesBackupResources sizes the Kubernetes strategy Pod. The dump is held + # in ephemeral storage while it moves and jq reads it into memory, so a + # cluster with a large etcd prefix needs more of both. + kubernetesBackupResources: + requests: + cpu: 10m + memory: 64Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi # The Kafka metadata strategy carries no client image here: unlike the Altinity # and RabbitMQ drivers, which need only curl (plus jq for Altinity), its driver # runs the Kafka Admin-API CLI (kafka-topics / kafka-configs), which only the @@ -105,10 +123,6 @@ backupStorage: endpoint: http://seaweedfs-s3.tenant-root.svc.cozy.local:8333 # region is re-projected into cozy-backups-creds on the next reconcile # (1-minute tick for system namespaces, on-demand for tenant namespaces). - # However, charts that consume the region via secretKeyRef in env vars — - # the chart-emitted ClickHouse sidecar on `useSystemBucket: true` is the - # only such consumer today — only pick up the new value on Pod restart. - # Plan a rollout of those Pods when you change this value. region: us-east-1 forcePathStyle: true # systemSecretName — the human-friendly Secret produced by the Bucket diff --git a/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml b/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml index 560bc4598..72f24e1f9 100644 --- a/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml +++ b/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml @@ -15,7 +15,13 @@ stringData: _namespace: host: {{ index .Values._cluster "root-host" | quote }} etcd: tenant-root + {{- /* Without ingress on the root Tenant there is no + ingress-nginx-system in tenant-root, so the key stays empty, as + it does for a sub-tenant with no ingress anywhere above it. A + missing key keeps tenant-root for Secrets written before it. */ -}} + {{- if eq (toString (index .Values._cluster "ingress-enabled" | default "true")) "true" }} ingress: tenant-root + {{- end }} {{- /* `gateway` is conditional, the other four `_namespace` keys are unconditional. The `apps/tenant` chart gates tenant-root via `ne (include "tenant.name" .) "tenant-root"`, so the namespace.yaml diff --git a/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml b/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml index 28a88b0e1..33db1dcd5 100644 --- a/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml +++ b/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml @@ -19,14 +19,16 @@ set: # never reaches tenant-root — this Secret is the only place that # value gets set for the root tenant. # -# Four of the five `_namespace.*` keys (`etcd`, `ingress`, -# `monitoring`, `seaweedfs`) are always rendered, because their -# dependent platform apps are always installable in tenant-root. -# `gateway` is conditional on `_cluster.gateway-enabled`: when the -# platform-level flag is false (the default), Gateway is not -# installed, and writing `gateway: tenant-root` would point apps at a -# Gateway that doesn't exist. The fixtures below pin both halves of -# that contract. +# Three of the five `_namespace.*` keys (`etcd`, `monitoring`, +# `seaweedfs`) are always rendered, because their dependent platform +# apps are always installable in tenant-root. `gateway` is conditional +# on `_cluster.gateway-enabled`: when the platform-level flag is false +# (the default), Gateway is not installed, and writing +# `gateway: tenant-root` would point apps at a Gateway that doesn't +# exist. `ingress` is conditional on `_cluster.ingress-enabled` for the +# same reason: with ingress off on the root Tenant there is no +# ingress-nginx-system in tenant-root, and seaweedfs would wait on it +# forever. The fixtures below pin both halves of each contract. tests: - it: omits _namespace.gateway when _cluster.gateway-enabled is missing (default) asserts: @@ -52,6 +54,54 @@ tests: path: stringData["values.yaml"] pattern: '(?m)^\s+gateway: tenant-root$' + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is missing (Secrets written before the key existed) + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is "true" + set: + _cluster: + ingress-enabled: "true" + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is a bool true + set: + _cluster: + ingress-enabled: true + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: omits _namespace.ingress when _cluster.ingress-enabled is "false" + set: + _cluster: + ingress-enabled: "false" + asserts: + - notMatchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress:' + + - it: keeps _namespace.{etcd,monitoring,seaweedfs} when ingress is off + set: + _cluster: + ingress-enabled: "false" + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+etcd: tenant-root$' + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+monitoring: tenant-root$' + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+seaweedfs: tenant-root$' + - it: keeps _namespace.{etcd,ingress,monitoring,seaweedfs} unconditional when Gateway is off — regression guard for a refactor that accidentally gates them on the same flag asserts: - matchRegex: diff --git a/packages/system/dashboard/images/console/Containerfile b/packages/system/dashboard/images/console/Containerfile index 84d634f46..d7493c60c 100644 --- a/packages/system/dashboard/images/console/Containerfile +++ b/packages/system/dashboard/images/console/Containerfile @@ -32,7 +32,14 @@ FROM nginxinc/nginx-unprivileged:1.29-alpine@sha256:0c79d56aee561a1d81c63f00eee5 COPY --from=builder /src/apps/console/dist /usr/share/nginx/html -COPY <<'EOF' /etc/nginx/conf.d/default.conf +# The config is a template the image's entrypoint renders into conf.d at +# startup, substituting only variables set in the environment: CLUSTER_DOMAIN, +# which the chart overrides with the platform cluster domain, and +# NGINX_LOCAL_RESOLVERS, which the entrypoint reads from the pod's resolv.conf. +ENV CLUSTER_DOMAIN=cozy.local +ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1 + +COPY <<'EOF' /etc/nginx/templates/default.conf.template map $http_upgrade $connection_upgrade { default upgrade; '' close; @@ -54,6 +61,35 @@ server { root /usr/share/nginx/html; index index.html; + # Logs and metrics roots of a namespace or of one application, served by + # the observability proxy rather than the API server. A regex location wins + # over the /apis prefix below. The upstream is a variable so nginx resolves + # it per request: on a cluster without the proxy nginx still starts, and a + # request it cannot deliver gets the not-installed answer below. The + # resolver does not apply search domains, hence the fully qualified name. + location ~ ^/apis/(?:cozystack\.io/v1alpha1/namespaces/[^/]+/(?:logs|metrics)|apps\.cozystack\.io/v1alpha1/namespaces/[^/]+/[^/]+/[^/]+/(?:log|metrics))/ { + resolver ${NGINX_LOCAL_RESOLVERS}; + set $observability_proxy observability-proxy.cozy-observability-proxy.svc.${CLUSTER_DOMAIN}:80; + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + # With proxy_intercept_errors off this catches only the 502 and 504 + # nginx makes itself when the proxy cannot be resolved or reached; + # the proxy's own 502 and 504 for an unreachable store pass through. + proxy_intercept_errors off; + error_page 502 504 = @observability_proxy_unavailable; + proxy_pass http://$observability_proxy; + } + + # The console reads a Kubernetes Status body as "not installed", the same + # answer kube-apiserver gives for a root nothing serves. + location @observability_proxy_unavailable { + default_type application/json; + return 404 '{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the observability proxy is not available on this cluster","reason":"NotFound","code":404}'; + } + # Kubernetes API proxy for both core and aggregated groups. # Chunked-encoding watches require proxy_buffering off and a long timeout. location /apis { diff --git a/packages/system/dashboard/templates/console.yaml b/packages/system/dashboard/templates/console.yaml index c4e641578..c05f3ee4b 100644 --- a/packages/system/dashboard/templates/console.yaml +++ b/packages/system/dashboard/templates/console.yaml @@ -40,6 +40,11 @@ spec: containers: - image: {{ .Values.console.image | quote }} imagePullPolicy: IfNotPresent + {{- with index .Values._cluster "cluster-domain" }} + env: + - name: CLUSTER_DOMAIN + value: {{ . | quote }} + {{- end }} livenessProbe: failureThreshold: 3 httpGet: diff --git a/packages/system/dashboard/tests/console_cluster_domain_test.yaml b/packages/system/dashboard/tests/console_cluster_domain_test.yaml new file mode 100644 index 000000000..c5e605638 --- /dev/null +++ b/packages/system/dashboard/tests/console_cluster_domain_test.yaml @@ -0,0 +1,42 @@ +suite: console cluster domain +# The console nginx names the observability proxy by its fully qualified +# Service name, because the resolver it uses for a per-request upstream does +# not apply the pod's search domains. The image defaults to cozy.local; the +# chart passes the platform cluster domain so a cluster on another domain +# still reaches the proxy. +templates: + - templates/console.yaml + - templates/configmap.yaml + +release: + name: dashboard + namespace: cozy-dashboard + +tests: + - it: passes the platform cluster domain to the console + template: templates/console.yaml + set: + _cluster: + cluster-domain: cluster.local + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLUSTER_DOMAIN + value: cluster.local + + - it: leaves the image default when the platform passes no cluster domain + template: templates/console.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # The entrypoint renders the nginx template into /etc/nginx/conf.d at + # startup. On a read-only root it logs the failure and starts nginx on the + # stock default.conf, which serves no /apis, /api or proxy route at all. + - it: keeps the root filesystem writable for the rendered nginx config + template: templates/console.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: false diff --git a/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml new file mode 100644 index 000000000..efdbd1158 --- /dev/null +++ b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml @@ -0,0 +1,167 @@ +{{- /* + The VictoriaMetrics operator puts its finalizer on the VMAgent and is the only + thing that ever removes it. The operator ships as a separate release that is + gated on the same addon flag, so disabling the addon removes both at once and + Flux does not order deletions: when the operator goes first, the VMAgent stays + Terminating and the uninstall never finishes. The other kinds are handled the + same way in case an operator version finalizes them too; for those the patch + is a no-op. + + So delete them before Helm does, give a live operator a bounded chance to + finalize them itself, and only then clear the finalizers by hand. The VMAgent's + Deployment, Service and Secrets carry owner references and are garbage + collected once it is gone. + + Cluster-scoped RBAC because the objects span two namespaces, and a Role in the + operator's namespace could not be created if that namespace were already + gone. The hook always exits 0: a cleanup that cannot complete must not be what + keeps the release from uninstalling. +*/ -}} +{{- $name := "vm-objects-cleanup" }} +{{- $img := printf "%s:%s" .Values.kubectlImage.repository .Values.kubectlImage.tag }} +{{- with .Values.kubectlImage.digest }}{{- $img = printf "%s@%s" $img . }}{{- end }} +{{- $kinds := list "vmagents" "vmrules" "vmservicescrapes" "vmpodscrapes" "vmnodescrapes" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +rules: + - apiGroups: ["operator.victoriametrics.com"] + resources: + {{- toYaml $kinds | nindent 6 }} + verbs: ["get", "list", "delete", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation +spec: + # Bounds every kubectl call as well: --request-timeout would make kubectl + # skip the in-cluster credentials (hack/check-hook-request-timeout.bats). + # Well inside the HelmRelease's 10m uninstall budget, and above the 60s + # wait plus one call per object on either side of it. + activeDeadlineSeconds: 180 + backoffLimit: 0 + ttlSecondsAfterFinished: 600 + template: + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + # The image user is the non-numeric `nonroot`, which the kubelet cannot + # check against runAsNonRoot. + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumes: + - name: home + emptyDir: {} + containers: + - name: cleanup + image: {{ $img | quote }} + imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # kubectl writes its discovery cache under $HOME. + env: + - name: HOME + value: /tmp + volumeMounts: + - name: home + mountPath: /tmp + command: + - /bin/sh + - -c + - | + rel={{ .Release.Name | quote }} + relns={{ .Release.Namespace | quote }} + kinds={{ join " " $kinds | quote }} + + # Selected by Helm's ownership annotations rather than by name, so + # the alert rules added under alerts/ are covered without a list + # to keep in step. A kind whose CRD is gone just prints an error. + owned() { + for k in $kinds; do + kubectl get "$k.operator.victoriametrics.com" -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.helm\.sh/release-name}{"|"}{.metadata.annotations.meta\.helm\.sh/release-namespace}{"|"}{.metadata.namespace}{"|"}{.metadata.name}{"\n"}{end}' | + while IFS='|' read -r r rns ns n; do + if [ "$r" = "$rel" ] && [ "$rns" = "$relns" ]; then + echo "$k.operator.victoriametrics.com $ns $n" + fi + done + done + } + + objs=$(owned) + if [ -z "$objs" ]; then + echo "no VictoriaMetrics objects left for $relns/$rel" + exit 0 + fi + + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" delete "$k" "$n" --wait=false + done + + deadline=$(( $(date +%s) + 60 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 2 + objs=$(owned) + if [ -z "$objs" ]; then + echo "the operator finalized every object" + exit 0 + fi + done + + echo "finalizers still held after 60s; clearing them by hand" + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" patch "$k" "$n" --type=merge \ + -p '{"metadata":{"finalizers":null}}' + done + exit 0 diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 00ab63c10..0eb6d9d80 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -10,7 +10,7 @@ spec: shardCount: 1 externalLabels: cluster: {{ .Values.vmagent.externalLabels.cluster }} - tenant: {{ .Values.global.target }} + tenant: {{ .Values.vmagent.externalLabels.tenant | default .Values.global.target | quote }} extraArgs: {{- toYaml (deepCopy .Values.vmagent.extraArgs | mergeOverwrite (fromYaml (include "monitoring-agents.vmagent.defaultExtraArgs" .))) | nindent 4 }} remoteWrite: @@ -20,6 +20,39 @@ spec: scrapeInterval: 30s selectAllByDefault: true + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional.yaml diff --git a/packages/system/monitoring-agents/templates/vpa.yaml b/packages/system/monitoring-agents/templates/vpa.yaml index 9a31fab94..1ac949f30 100644 --- a/packages/system/monitoring-agents/templates/vpa.yaml +++ b/packages/system/monitoring-agents/templates/vpa.yaml @@ -6,6 +6,11 @@ Initial sets requests at creation without evicting, removing the trigger. Do not switch back to Auto without fixing the underlying Cilium IP-reuse race. */ -}} +{{- /* + Rendered only when the CRD is served: a release manifest that records a VPA + the cluster cannot map is one Helm can no longer uninstall. +*/ -}} +{{- if .Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler" }} apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: @@ -34,3 +39,4 @@ spec: maxAllowed: cpu: 4000m memory: 6G +{{- end }} diff --git a/packages/system/monitoring-agents/tests/uninstall_test.yaml b/packages/system/monitoring-agents/tests/uninstall_test.yaml new file mode 100644 index 000000000..df8a3eaa6 --- /dev/null +++ b/packages/system/monitoring-agents/tests/uninstall_test.yaml @@ -0,0 +1,141 @@ +suite: the release uninstalls whatever order its dependencies go in + +# Disabling the addon removes this release, the VictoriaMetrics operator and the +# VPA CRDs together, and Flux deletes them in parallel. When the operator goes +# first, nothing clears its finalizer on the objects this chart created and the +# uninstall hangs; when the VPA CRD goes first, a release manifest recording a +# VPA cannot be built and the uninstall fails. +release: + name: cozy-monitoring-agents + namespace: cozy-monitoring +templates: + - templates/vm-cleanup-hook.yaml + - templates/vpa.yaml +tests: + - it: runs before Helm deletes anything, bounded and with one attempt + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.namespace + value: cozy-monitoring + - equal: + path: spec.activeDeadlineSeconds + value: 180 + - equal: + path: spec.backoffLimit + value: 0 + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: ^docker\.io/clastix/kubectl:v[0-9.]+@sha256:[0-9a-f]{64}$ + + - it: its RBAC exists only for the hook and is removed with it + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "-10" + - matchRegex: + path: metadata.annotations["helm.sh/hook-delete-policy"] + pattern: hook-succeeded + + - it: may delete and clear finalizers on every VictoriaMetrics kind the chart renders + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: rules[0].apiGroups + value: ["operator.victoriametrics.com"] + - contains: + path: rules[0].resources + content: vmagents + - contains: + path: rules[0].resources + content: vmrules + - contains: + path: rules[0].resources + content: vmservicescrapes + - contains: + path: rules[0].resources + content: vmpodscrapes + - contains: + path: rules[0].resources + content: vmnodescrapes + - equal: + path: rules[0].verbs + value: ["get", "list", "delete", "patch"] + - lengthEqual: + path: rules + count: 1 + + - it: grants that role to the hook's own service account + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRoleBinding + asserts: + - equal: + path: roleRef.name + value: cozy-monitoring-vm-objects-cleanup + - equal: + path: subjects[0] + value: + kind: ServiceAccount + name: vm-objects-cleanup + namespace: cozy-monitoring + + - it: touches only objects this release owns, and clears their finalizers + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: rel="cozy-monitoring-agents" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: relns="cozy-monitoring" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: meta\\\.helm\\\.sh/release-name + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'delete "\$k" "\$n" --wait=false' + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '"finalizers":null' + # A pre-delete hook that fails aborts the uninstall it exists to unblock. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: exit 0\s*$ + + - it: the VPA is rendered when its CRD is served + template: templates/vpa.yaml + capabilities: + apiVersions: + - autoscaling.k8s.io/v1/VerticalPodAutoscaler + asserts: + - hasDocuments: + count: 1 + - isKind: + of: VerticalPodAutoscaler + + - it: the VPA stays out of the release manifest when its CRD is not served + template: templates/vpa.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..4e4b93385 --- /dev/null +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,77 @@ +suite: cluster-wide vmagent stamps application identity on exporter series +templates: + - templates/vmagent.yaml +release: + name: monitoring-agents + namespace: cozy-monitoring + +# The tenant-resource-metrics proxy scopes a query to one application by +# apps_cozystack_io_application_{group,kind,name}. The labels are only a +# boundary if the scraped pod cannot choose them: the class sets all three +# on every target it covers (a placeholder when the pod has no lineage), so with +# honorLabels false an exporter's own copy is renamed to exported_* and +# dropped. The same rules live in the tenant vmagent of the monitoring +# package and are pinned there by the same assertions. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" diff --git a/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml b/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml new file mode 100644 index 000000000..894a1c053 --- /dev/null +++ b/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml @@ -0,0 +1,65 @@ +suite: vmagent remote write targets and tenant label + +# The root bundle sets only global.target, and the chart defaults derive both +# the remote write URLs and the tenant label from it. A tenant Kubernetes +# cluster gets this chart through packages/apps/kubernetes instead, which +# passes vmagent.remoteWrite.urls and vmagent.externalLabels.tenant and leaves +# global.target at its default. If either value is dropped, vmagent writes to +# cozy-monitoring services that do not exist in the tenant cluster, or labels +# every series tenant="cozy-monitoring". +templates: + - templates/vmagent.yaml +release: + name: cozy-monitoring-agents + namespace: cozy-monitoring +tests: + - it: root install writes to both stores of global.target and labels series with it + set: + global.target: tenant-root + asserts: + - equal: + path: spec.remoteWrite + value: + - url: http://vminsert-shortterm.tenant-root.svc:8480/insert/0/prometheus + - url: http://vminsert-longterm.tenant-root.svc:8480/insert/0/prometheus + - equal: + path: spec.externalLabels.tenant + value: tenant-root + + - it: tenant cluster writes only to the URLs its parent app passes + set: + vmagent: + externalLabels: + cluster: kubernetes-clusterprod + tenant: tenant-a + remoteWrite: + urls: + - http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus + asserts: + - equal: + path: spec.remoteWrite + value: + - url: http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus + + - it: tenant cluster labels series with the owning tenant, not global.target + set: + vmagent: + externalLabels: + cluster: kubernetes-clusterprod + tenant: tenant-a + asserts: + - equal: + path: spec.externalLabels + value: + cluster: kubernetes-clusterprod + tenant: tenant-a + + - it: tenant label stays a string when the value reads as a YAML scalar + set: + vmagent: + externalLabels: + tenant: "true" + asserts: + - equal: + path: spec.externalLabels.tenant + value: "true" diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index d7dd870b0..b6c046ea5 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -575,3 +575,15 @@ prometheus-node-exporter: memory: 32Mi limits: memory: 256Mi + +# Image for the pre-delete hook that releases the VictoriaMetrics objects (see +# templates/vm-cleanup-hook.yaml). Only needs a kubectl binary. Digest-pinned so +# an upstream retag does not change what runs across the fleet; refresh with +# `docker manifest inspect docker.io/clastix/kubectl:v1.32`. Same pin as the +# etcd-operator hook. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +kubectlImage: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + pullPolicy: IfNotPresent diff --git a/packages/system/monitoring/templates/grafana/grafana.yaml b/packages/system/monitoring/templates/grafana/grafana.yaml index 85517151a..2da6d2fc6 100644 --- a/packages/system/monitoring/templates/grafana/grafana.yaml +++ b/packages/system/monitoring/templates/grafana/grafana.yaml @@ -280,7 +280,7 @@ spec: path: auth.ini {{- end }} {{- end }} - {{- if ne $gatewayEnabled "true" }} + {{- if and (ne $gatewayEnabled "true") $ingress }} ingress: {{- /* Operator wildcard mode: nginx serves the default SSL cert, so skip per-host ACME. */}} {{- if not $wildcardSecret }} diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index 98bff1e14..919fdef63 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -24,6 +24,39 @@ spec: scrapeInterval: 30s selectAllByDefault: false + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} {{- with .Values.vmagent.inlineScrapeConfig }} inlineScrapeConfig: | {{- . | nindent 4 }} diff --git a/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml b/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml new file mode 100644 index 000000000..9e6b0350c --- /dev/null +++ b/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml @@ -0,0 +1,40 @@ +suite: grafana ingress needs an ingress class to name +templates: + - templates/grafana/grafana.yaml + +release: + name: monitoring + namespace: tenant-root + +# An empty _namespace.ingress means no ingress-nginx serves this tenant, and +# the API rejects the ingressClassName "" the operator would put on the +# Ingress it creates from spec.ingress. + +tests: + - it: leaves spec.ingress out when the tenant has no ingress + set: + _cluster: + issuer-name: letsencrypt-prod + gateway-enabled: "false" + _namespace: + host: example.org + ingress: "" + asserts: + - equal: + path: kind + value: Grafana + - notExists: + path: spec.ingress + + - it: keeps spec.ingress when the tenant has an ingress + set: + _cluster: + issuer-name: letsencrypt-prod + gateway-enabled: "false" + _namespace: + host: example.org + ingress: tenant-root + asserts: + - equal: + path: spec.ingress.spec.ingressClassName + value: tenant-root diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..3a77fe6ee --- /dev/null +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,73 @@ +suite: tenant vmagent stamps application identity on exporter series +templates: + - templates/vm/vmagent.yaml +release: + name: monitoring + namespace: tenant-foo + +# Same rules as the cluster-wide vmagent in monitoring-agents, pinned there +# by the same assertions; see that suite for why the placeholder and the +# drop exist. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" diff --git a/packages/system/observability-proxy/Chart.yaml b/packages/system/observability-proxy/Chart.yaml new file mode 100644 index 000000000..282c2234a --- /dev/null +++ b/packages/system/observability-proxy/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-observability-proxy +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/observability-proxy/Makefile b/packages/system/observability-proxy/Makefile new file mode 100644 index 000000000..ee66c0656 --- /dev/null +++ b/packages/system/observability-proxy/Makefile @@ -0,0 +1,20 @@ +NAME=observability-proxy +NAMESPACE=cozy-observability-proxy + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +test: + helm unittest . + +image: image-observability-proxy + +image-observability-proxy: + docker buildx build -f images/observability-proxy/Dockerfile ../../.. \ + $(call image-tags,observability-proxy,v$(COZYSTACK_VERSION)) \ + $(call cache-args,observability-proxy) \ + --metadata-file images/observability-proxy.json \ + $(BUILDX_ARGS) + IMAGE="$(REGISTRY)/observability-proxy:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/observability-proxy.json -o json -r)" \ + yq -i '.observabilityProxy.image = strenv(IMAGE)' values.yaml + rm -f images/observability-proxy.json diff --git a/packages/system/observability-proxy/images/observability-proxy/Dockerfile b/packages/system/observability-proxy/images/observability-proxy/Dockerfile new file mode 100644 index 000000000..4615e74ec --- /dev/null +++ b/packages/system/observability-proxy/images/observability-proxy/Dockerfile @@ -0,0 +1,25 @@ +FROM --platform=$BUILDPLATFORM golang:1.27-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS builder +ARG GOPROXY + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace + +COPY go.mod go.sum ./ +RUN for attempt in 1 2 3; do GOOS=$TARGETOS GOARCH=$TARGETARCH go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done + +COPY api api/ +COPY pkg pkg/ +COPY cmd cmd/ +COPY internal internal/ + +RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -ldflags="-extldflags=-static" -o /observability-proxy ./cmd/observability-proxy + +FROM scratch + +COPY --from=builder /observability-proxy /observability-proxy +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +USER 65532:65532 +ENTRYPOINT ["/observability-proxy"] diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml new file mode 100644 index 000000000..581c15fde --- /dev/null +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -0,0 +1,68 @@ +{{- $clusterDomain := index (.Values._cluster | default dict) "cluster-domain" | default "cozy.local" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: observability-proxy + labels: + app.kubernetes.io/name: observability-proxy +spec: + replicas: {{ .Values.observabilityProxy.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: observability-proxy + template: + metadata: + labels: + app.kubernetes.io/name: observability-proxy + spec: + serviceAccountName: observability-proxy + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: observability-proxy + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: observability-proxy + image: {{ .Values.observabilityProxy.image | quote }} + args: + - --listen-address=:8080 + - --probe-address=:8081 + - --cluster-domain={{ $clusterDomain }} + - --root-tenant={{ .Values.observabilityProxy.rootTenant }} + - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: probes + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: probes + initialDelaySeconds: 15 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + {{- with .Values.observabilityProxy.resources }} + resources: {{- toYaml . | nindent 10 }} + {{- end }} diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml new file mode 100644 index 000000000..3c5392903 --- /dev/null +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy:observability-proxy +rules: +# Every request is authenticated and authorized against the API server. +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +# The namespace.cozystack.io/monitoring label names the logs store; it is +# read from an informer cache, which only lists and watches. +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cozy:observability-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:observability-proxy +subjects: +- kind: ServiceAccount + name: observability-proxy + namespace: {{ .Release.Namespace }} diff --git a/packages/system/observability-proxy/templates/sa.yaml b/packages/system/observability-proxy/templates/sa.yaml new file mode 100644 index 000000000..5a4eb2182 --- /dev/null +++ b/packages/system/observability-proxy/templates/sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: observability-proxy diff --git a/packages/system/observability-proxy/templates/service.yaml b/packages/system/observability-proxy/templates/service.yaml new file mode 100644 index 000000000..6a398532a --- /dev/null +++ b/packages/system/observability-proxy/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: observability-proxy + labels: + app.kubernetes.io/name: observability-proxy +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + app.kubernetes.io/name: observability-proxy diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml new file mode 100644 index 000000000..23460352f --- /dev/null +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -0,0 +1,14 @@ +# Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). +# No API server serves cozystack.io logs: the rule exists only for the +# SubjectAccessReview the observability proxy asks before it answers a read +# under /apis/cozystack.io/v1alpha1/namespaces//logs/. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy:observability:view + labels: + rbac.cozystack.io/aggregate-to-tenant-view: "true" +rules: +- apiGroups: ["cozystack.io"] + resources: ["logs"] + verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml new file mode 100644 index 000000000..ddf9d34b8 --- /dev/null +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -0,0 +1,109 @@ +suite: observability-proxy Deployment +# The flags, ports and pod label below are the contract the proxy binary, the +# Service and the console nginx are written against. A rename on either side +# breaks routing without failing a render. +templates: + - templates/deployment.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: labels the pods with the name the Service selects on + asserts: + - isKind: + of: Deployment + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: observability-proxy + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: observability-proxy + - equal: + path: spec.template.spec.serviceAccountName + value: observability-proxy + + - it: passes the listen, probe, root tenant and auth cache flags + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --listen-address=:8080 + - contains: + path: spec.template.spec.containers[0].args + content: --probe-address=:8081 + - contains: + path: spec.template.spec.containers[0].args + content: --root-tenant=tenant-root + - contains: + path: spec.template.spec.containers[0].args + content: --auth-cache-ttl=10s + + - it: falls back to cozy.local when the platform passes no cluster domain + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cozy.local + + - it: passes the platform cluster domain + set: + _cluster: + cluster-domain: cluster.local + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cluster.local + - notContains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cozy.local + + - it: exposes the API and probe ports + asserts: + - equal: + path: spec.template.spec.containers[0].ports + value: + - name: http + containerPort: 8080 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + + - it: probes /healthz and /readyz on the probe port + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet + value: + path: /healthz + port: probes + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet + value: + path: /readyz + port: probes + + - it: runs from the image in values + set: + observabilityProxy.image: registry.example/observability-proxy:dev@sha256:0000 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/observability-proxy:dev@sha256:0000 + + - it: satisfies the restricted pod security profile + asserts: + - equal: + path: spec.template.spec.securityContext + value: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + - equal: + path: spec.template.spec.containers[0].securityContext + value: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml new file mode 100644 index 000000000..af339b5d0 --- /dev/null +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -0,0 +1,67 @@ +suite: observability-proxy RBAC +# The proxy authenticates and authorizes every request through the API server, +# so a dropped verb here turns every request into an error at runtime rather +# than a render failure. The tenant role is what lets a namespace viewer pass +# the SubjectAccessReview the proxy asks for the namespace logs root. +templates: + - templates/rbac.yaml + - templates/tenant-clusterroles.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: lets the proxy review tokens and access, and list and watch namespaces + template: templates/rbac.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: rules + value: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] + + - it: binds the proxy role to the proxy ServiceAccount in the release namespace + template: templates/rbac.yaml + documentSelector: + path: kind + value: ClusterRoleBinding + asserts: + - equal: + path: roleRef + value: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:observability-proxy + - equal: + path: subjects + value: + - kind: ServiceAccount + name: observability-proxy + namespace: cozy-observability-proxy + + - it: aggregates list on cozystack.io logs into the tenant view role + template: templates/tenant-clusterroles.yaml + asserts: + - isKind: + of: ClusterRole + - equal: + path: metadata.labels + value: + rbac.cozystack.io/aggregate-to-tenant-view: "true" + - equal: + path: rules + value: + - apiGroups: ["cozystack.io"] + resources: ["logs"] + verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/service_test.yaml b/packages/system/observability-proxy/tests/service_test.yaml new file mode 100644 index 000000000..123cc01ae --- /dev/null +++ b/packages/system/observability-proxy/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: observability-proxy Service +# The console nginx proxies to observability-proxy..svc on port 80, which +# must reach the pods' 8080, so the name, the port and the selector are pinned +# here. +templates: + - templates/service.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: serves port 80 named http on the pods' http port + asserts: + - isKind: + of: Service + - equal: + path: metadata.name + value: observability-proxy + - equal: + path: spec.ports + value: + - name: http + port: 80 + targetPort: http + protocol: TCP + - equal: + path: spec.selector + value: + app.kubernetes.io/name: observability-proxy diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml new file mode 100644 index 000000000..919e450a9 --- /dev/null +++ b/packages/system/observability-proxy/values.yaml @@ -0,0 +1,16 @@ +# Injected by the platform from the cozystack-values Secret. The empty +# default keeps the chart renderable on its own, outside a cluster. +_cluster: {} + +observabilityProxy: + image: "ghcr.io/cozystack/cozystack/observability-proxy:v0.0.0" + replicas: 2 + rootTenant: tenant-root + authCacheTTL: 10s + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 39adc0f58..0cd63edd2 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -30,7 +30,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: diff --git a/packages/system/seaweedfs-db/templates/database.yaml b/packages/system/seaweedfs-db/templates/database.yaml index 807bd13af..136a3be83 100644 --- a/packages/system/seaweedfs-db/templates/database.yaml +++ b/packages/system/seaweedfs-db/templates/database.yaml @@ -11,7 +11,7 @@ spec: storage: size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} monitoring: diff --git a/packages/system/seaweedfs-db/tests/database_test.yaml b/packages/system/seaweedfs-db/tests/database_test.yaml index 3eff5a112..2e6244cf1 100644 --- a/packages/system/seaweedfs-db/tests/database_test.yaml +++ b/packages/system/seaweedfs-db/tests/database_test.yaml @@ -60,3 +60,17 @@ tests: - equal: path: spec.inheritedMetadata.labels["policy.cozystack.io/allow-to-apiserver"] value: "true" + + # A StorageClass name may be all digits; emitted unquoted it renders as a + # number, which the Cluster rejects. + - it: keeps a numeric storage class name a string + release: + name: seaweedfs-db + namespace: cozy-seaweedfs + set: + db: + storageClass: "123" + asserts: + - equal: + path: spec.storage.storageClass + value: "123" diff --git a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml index 1fa5ad336..6cafe5c5d 100644 --- a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml +++ b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml @@ -8,7 +8,7 @@ spec: singular: seaweedfs plural: seaweedfses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"lifecycle":{"description":"S3 bucket lifecycle configuration.","type":"object","default":{},"properties":{"enabled":{"description":"Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new.","type":"boolean","default":false},"storageClass":{"description":"StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again.","type":"string","default":"","maxLength":253,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$"}}}}} release: prefix: "" labels: @@ -26,7 +26,7 @@ spec: description: Seaweedfs module: true icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTEzOC42ODUgMTIxLjA1N0MxMzguNjg1IDEyNi42NTIgMTM2LjQ2MiAxMzIuMDE3IDEzMi41MDQgMTM1Ljk3M0MxMjguNTQ3IDEzOS45MjkgMTIzLjE3OSAxNDIuMTUxIDExNy41ODIgMTQyLjE1MUMxMTEuOTg1IDE0Mi4xNTEgMTA2LjYxOCAxMzkuOTI5IDEwMi42NiAxMzUuOTczQzk4LjcwMjggMTMyLjAxNyA5Ni40Nzk1IDEyNi42NTIgOTYuNDc5NSAxMjEuMDU3Qzk2LjQ3OTUgMTE4LjI4NyA5Ny4wMjUzIDExNS41NDQgOTguMDg1OCAxMTIuOTg1Qzk5LjE0NjMgMTEwLjQyNSAxMDAuNzAxIDEwOC4xIDEwMi42NiAxMDYuMTQxQzEwNC42MiAxMDQuMTgyIDEwNi45NDYgMTAyLjYyOSAxMDkuNTA3IDEwMS41NjlDMTEyLjA2NyAxMDAuNTA5IDExNC44MTEgOTkuOTYyOSAxMTcuNTgyIDk5Ljk2MjlDMTIwLjM1MyA5OS45NjI5IDEyMy4wOTggMTAwLjUwOSAxMjUuNjU4IDEwMS41NjlDMTI4LjIxOCAxMDIuNjI5IDEzMC41NDQgMTA0LjE4MiAxMzIuNTA0IDEwNi4xNDFDMTM0LjQ2NCAxMDguMSAxMzYuMDE4IDExMC40MjUgMTM3LjA3OSAxMTIuOTg1QzEzOC4xMzkgMTE1LjU0NCAxMzguNjg1IDExOC4yODcgMTM4LjY4NSAxMjEuMDU3WiIgZmlsbD0idXJsKCNwYWludDFfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMTEwLjMzNiAxMjYuMTQ3SDEyMy42OFYxMzYuODIzSDExMC4zMzZWMTI2LjE0N1oiIGZpbGw9IiMwRjVFOUMiLz4KPHBhdGggZD0iTTExOS4yNyAxMTIuMjk0QzExOS4yNyAxMTIuNzE0IDExOS4xNzggMTEzLjEzMSAxMTkgMTEzLjUxOUMxMTguODIxIDExMy45MDggMTE4LjU2IDExNC4yNjEgMTE4LjIzIDExNC41NThDMTE3LjkwMSAxMTQuODU1IDExNy41MDkgMTE1LjA5MSAxMTcuMDc5IDExNS4yNTJDMTE2LjY0OCAxMTUuNDEzIDExNi4xODYgMTE1LjQ5NiAxMTUuNzIgMTE1LjQ5NkMxMTQuNzc4IDExNS40OTYgMTEzLjg3NSAxMTUuMTU4IDExMy4yMSAxMTQuNTU4QzExMi41NDQgMTEzLjk1NyAxMTIuMTcgMTEzLjE0MyAxMTIuMTcgMTEyLjI5NEMxMTIuMTcgMTExLjQ0NSAxMTIuNTQ0IDExMC42MyAxMTMuMjEgMTEwLjAzQzExMy44NzUgMTA5LjQyOSAxMTQuNzc4IDEwOS4wOTIgMTE1LjcyIDEwOS4wOTJDMTE2LjE4NiAxMDkuMDkyIDExNi42NDggMTA5LjE3NSAxMTcuMDc5IDEwOS4zMzZDMTE3LjUwOSAxMDkuNDk2IDExNy45MDEgMTA5LjczMiAxMTguMjMgMTEwLjAzQzExOC41NiAxMTAuMzI3IDExOC44MjEgMTEwLjY4IDExOSAxMTEuMDY4QzExOS4xNzggMTExLjQ1NyAxMTkuMjcgMTExLjg3MyAxMTkuMjcgMTEyLjI5NFoiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyOC4xMSAxMTQuMTAzQzEyOC4xMSAxMTQuOTUzIDEyNy43MzYgMTE1Ljc2NyAxMjcuMDcgMTE2LjM2OEMxMjYuNDA0IDExNi45NjggMTI1LjUwMSAxMTcuMzA1IDEyNC41NiAxMTcuMzA1QzEyNC4wOTQgMTE3LjMwNSAxMjMuNjMyIDExNy4yMjMgMTIzLjIwMSAxMTcuMDYyQzEyMi43NzEgMTE2LjkwMSAxMjIuMzc5IDExNi42NjUgMTIyLjA1IDExNi4zNjhDMTIxLjcyIDExNi4wNyAxMjEuNDU4IDExNS43MTcgMTIxLjI4IDExNS4zMjlDMTIxLjEwMiAxMTQuOTQgMTIxLjAxIDExNC41MjQgMTIxLjAxIDExNC4xMDNDMTIxLjAxIDExMy42ODMgMTIxLjEwMiAxMTMuMjY2IDEyMS4yOCAxMTIuODc4QzEyMS40NTggMTEyLjQ5IDEyMS43MiAxMTIuMTM3IDEyMi4wNSAxMTEuODM5QzEyMi4zNzkgMTExLjU0MiAxMjIuNzcxIDExMS4zMDYgMTIzLjIwMSAxMTEuMTQ1QzEyMy42MzIgMTEwLjk4NCAxMjQuMDk0IDExMC45MDEgMTI0LjU2IDExMC45MDFDMTI1LjUwMSAxMTAuOTAxIDEyNi40MDQgMTExLjIzOSAxMjcuMDcgMTExLjgzOUMxMjcuNzM2IDExMi40NCAxMjguMTEgMTEzLjI1NCAxMjguMTEgMTE0LjEwM1oiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyMi4zMzMgMTE4Ljk3NkMxMjIuMzMzIDExOS44MjYgMTIxLjk1OCAxMjAuNjQgMTIxLjI5MyAxMjEuMjQxQzEyMC42MjcgMTIxLjg0MSAxMTkuNzI0IDEyMi4xNzggMTE4Ljc4MiAxMjIuMTc4QzExOC4zMTYgMTIyLjE3OCAxMTcuODU1IDEyMi4wOTYgMTE3LjQyNCAxMjEuOTM1QzExNi45OTMgMTIxLjc3NCAxMTYuNjAyIDEyMS41MzggMTE2LjI3MiAxMjEuMjQxQzExNS45NDMgMTIwLjk0MyAxMTUuNjgxIDEyMC41OSAxMTUuNTAzIDEyMC4yMDJDMTE1LjMyNCAxMTkuODEzIDExNS4yMzIgMTE5LjM5NyAxMTUuMjMyIDExOC45NzZDMTE1LjIzMiAxMTguNTU2IDExNS4zMjQgMTE4LjE0IDExNS41MDMgMTE3Ljc1MUMxMTUuNjgxIDExNy4zNjMgMTE1Ljk0MyAxMTcuMDEgMTE2LjI3MiAxMTYuNzEyQzExNi42MDIgMTE2LjQxNSAxMTYuOTkzIDExNi4xNzkgMTE3LjQyNCAxMTYuMDE4QzExNy44NTUgMTE1Ljg1NyAxMTguMzE2IDExNS43NzQgMTE4Ljc4MiAxMTUuNzc0QzExOS43MjQgMTE1Ljc3NCAxMjAuNjI3IDExNi4xMTIgMTIxLjI5MyAxMTYuNzEyQzEyMS45NTggMTE3LjMxMyAxMjIuMzMzIDExOC4xMjcgMTIyLjMzMyAxMTguOTc2WiIgZmlsbD0iIzU5Njg2RiIvPgo8cGF0aCBkPSJNMTE1LjMwOCAxMjEuOTA1QzExMy43MzUgMTIxLjQyNiAxMTUuNzA3IDEyMC42OCAxMTUuNDI5IDEyMC41NzNDMTE0LjY1MyAxMjAuMjc2IDExNS43MyAxMTkuMzMzIDExNi43NCAxMTguNTM5QzExNy42MjggMTE3Ljg0MSAxMTcuNjU5IDExNy44MzkgMTE3Ljg5MiAxMTguNDY4QzExOC4wMjQgMTE4LjgyMyAxMTguMzcxIDExOS4yMDYgMTE4LjY2NSAxMTkuMzE5QzExOS4yODkgMTE5LjU1OCAxMjAuOTUxIDExOS4yNjkgMTIwLjk1MSAxMTguODM1QzEyMC45NTEgMTE4LjIyMyAxMjEuNDE1IDExOC42OTkgMTIxLjc5NCAxMTkuNTMxQzEyMi40NTcgMTIwLjk4NyAxMjIuNDM3IDEyMi40NzIgMTIxLjQ1IDEyMi40NzJDMTIwLjg1OSAxMjIuNDcyIDEyMC4zMSAxMjIuNjkxIDEyMC4xOSAxMjMuMTQ3QzExNS4wNDYgMTI0LjYgMTE2LjQ3MSAxMjMuMjg0IDExNS4zMDggMTIxLjkwNVpNMTIzLjA5NCAxMTguMDU0QzEyMS42MDkgMTE3LjQ2NiAxMjAuNTQ3IDExNC44MzggMTIwLjU0NyAxMTMuMzA5QzEyMC41NDcgMTExLjMyMyAxMjIuNTQxIDEwOS4wOTUgMTI0LjMxNyAxMDkuMDk1QzEyOC4zMTUgMTA5LjA5NSAxMzAuNjg0IDExMi4yNjEgMTI4LjgzOCAxMTguNDA5QzEyOC4zODUgMTE5LjkxOSAxMjMuOTMzIDExOC4zODcgMTIzLjA5NCAxMTguMDU0Wk0xMjUuODY2IDExNS42NDlDMTI3LjU0NCAxMTQuNDc0IDEyNS44NTcgMTExLjc1NSAxMjMuOTgxIDExMi42MUMxMjIuNDc5IDExMy4yOTQgMTIzLjExOSAxMTYuMTE1IDEyNC43NyAxMTYuMTY0QzEyNC45NjEgMTE2LjE2OSAxMjUuNDU0IDExNS45MzggMTI1Ljg2NiAxMTUuNjQ5Wk0xMjQuMzM5IDExNC41NzlDMTIzLjc3MSAxMTQuMzgxIDEyMy44MDMgMTEzLjI1NiAxMjQuMzgzIDExMy4wMzRDMTI0LjkwMiAxMTIuODM1IDEyNS42MDQgMTEzLjM5OCAxMjUuNjA0IDExNC4wMTRDMTI1LjYwNCAxMTQuNDg5IDEyNC45MzYgMTE0Ljc4NyAxMjQuMzM5IDExNC41NzlaTTExMi4zMTkgMTE2Ljk4QzEwOS45NiAxMTUuNjcgMTEwLjI4NiAxMTEuMDA2IDExMi40MTcgMTA4Ljk2NUMxMTQuMzk2IDEwNy4wNjkgMTE4Ljc3OCAxMDcuMTAzIDExOS43NjQgMTA5LjQ2MkMxMjAuNDE1IDExMS4wMjEgMTIwLjIyOCAxMTIuNiAxMTkuMzk4IDExNC4wNzdDMTE4LjE5NCAxMTYuMjE5IDExNC4yNDIgMTE4LjA0OSAxMTIuMzE5IDExNi45OFpNMTE2LjU2IDExMy41OTNDMTE3LjMyNSAxMTIuOTAxIDExNy4yOTcgMTEyLjA2IDExNi41NzIgMTExLjI1OUMxMTUuNzc3IDExMC4zNzkgMTE0LjY5MiAxMTAuMzQ1IDExNC4wMzUgMTExLjM0N0MxMTMuNTI0IDExMi4xMjcgMTEzLjQzMSAxMTIuNTI4IDExMy45NDMgMTEzLjMwOUMxMTQuNTggMTE0LjI4IDExNS42NyAxMTQuMzk5IDExNi41NiAxMTMuNTkzWk0xMTQuMzc0IDExMi4zNEMxMTQuMTI4IDExMS42OTggMTE0Ljk4NSAxMTAuOTgxIDExNS42OCAxMTEuMjQ3QzExNS45NzQgMTExLjM2IDExNi4xNjQgMTExLjcxOCAxMTYuMTAyIDExMi4wNDRDMTE1Ljk2MSAxMTIuNzg1IDExNC42MzIgMTEzLjAxMiAxMTQuMzc0IDExMi4zNFoiIGZpbGw9IiNEM0Q2REEiLz4KPHBhdGggZD0iTTExOC40NjMgMTIxLjAwOEwxMTcuOTQ1IDEyMC43OEMxMTcuNTEgMTIwLjY4NSAxMTcuMjMxIDEyMS4xMDcgMTE3LjEzNiAxMjEuNTQzTDExNi44ODEgMTIyLjcwOUMxMTYuNzg2IDEyMy4xNDUgMTE3LjA2MSAxMjMuNTcxIDExNy40OTYgMTIzLjY2NkwxMTguNDQ3IDEyMy44NzRDMTE4Ljg4MiAxMjMuOTY5IDExOS4zMTEgMTIzLjY5NCAxMTkuNDA2IDEyMy4yNTlMMTE5LjY4NSAxMjEuNTU2QzExOS43OCAxMjEuMTIgMTE5LjQ3OSAxMjEuMjI4IDExOS4wNDMgMTIxLjEzM0wxMTguNjI4IDEyMS4wNDNDMTE4LjU3MyAxMjEuMzIxIDExOC41MTYgMTIxLjYxMyAxMTguNDcxIDEyMS44MzNDMTE4LjQzOCAxMjEuOTk4IDExOC40MDcgMTIyLjE0OCAxMTguMzc5IDEyMi4yODJDMTE4LjM1MSAxMjIuNDE1IDExOC4zMjcgMTIyLjUzMyAxMTguMzA1IDEyMi42MzVDMTE4LjI4MiAxMjIuNzM4IDExOC4yNjIgMTIyLjgyNSAxMTguMjQ1IDEyMi44OTdDMTE4LjIyOCAxMjIuOTY5IDExOC4yMTQgMTIzLjAyNSAxMTguMjAyIDEyMy4wNjhDMTE4LjE5NiAxMjMuMDg5IDExOC4xOTMgMTIzLjEwNyAxMTguMTg4IDEyMy4xMjFDMTE4LjE4MyAxMjMuMTM1IDExOC4xNzggMTIzLjE0NSAxMTguMTc1IDEyMy4xNTJDMTE4LjE3MyAxMjMuMTU1IDExOC4xNzIgMTIzLjE1NiAxMTguMTcxIDEyMy4xNThDMTE4LjE3IDEyMy4xNTkgMTE4LjE2OCAxMjMuMTYgMTE4LjE2NyAxMjMuMTZMMTE4LjE2NSAxMjMuMTU4QzExOC4xNjQgMTIzLjE1NiAxMTguMTYzIDEyMy4xNTIgMTE4LjE2MyAxMjMuMTQ4QzExOC4xNjIgMTIzLjE0IDExOC4xNjIgMTIzLjEyOSAxMTguMTYzIDEyMy4xMTVDMTE4LjE2MyAxMjMuMSAxMTguMTYzIDEyMy4wODMgMTE4LjE2NSAxMjMuMDYxQzExOC4xNjggMTIzLjAxOCAxMTguMTc1IDEyMi45NjEgMTE4LjE4MyAxMjIuODkxQzExOC4xOTIgMTIyLjgyIDExOC4yMDMgMTIyLjczNiAxMTguMjE2IDEyMi42NEMxMTguMjI5IDEyMi41NDMgMTE4LjI0NiAxMjIuNDMxIDExOC4yNjQgMTIyLjMwOEMxMTguMjgxIDEyMi4xODUgMTE4LjMwMSAxMjIuMDUxIDExOC4zMjMgMTIxLjkwM0MxMTguMzQ2IDEyMS43NTUgMTE4LjM3IDEyMS41OTIgMTE4LjM5NyAxMjEuNDE5QzExOC40MTYgMTIxLjI5OSAxMTguNDQyIDEyMS4xNCAxMTguNDYzIDEyMS4wMDhaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDMuNTgxIDExNi4zNjdDMTAzLjQ3OSAxMTYuMTAzIDEwMy42MjIgMTE1LjY5OSAxMDMuODk4IDExNS40NjlDMTA0LjMwMyAxMTUuMTMzIDEwNC41MDQgMTE1LjE1NSAxMDQuOTMgMTE1LjU4MUMxMDUuMjIgMTE1Ljg3MSAxMDUuMzU1IDExNi4yNzYgMTA1LjIzIDExNi40NzlDMTA0LjkwMiAxMTcuMDA5IDEwMy43OTggMTE2LjkzNCAxMDMuNTgxIDExNi4zNjdaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDExMi4yODFDMTA1LjY4IDExMS44NDkgMTA1LjcwMiAxMTEuNjYgMTA2LjE2NSAxMTEuMTk3QzEwNi42ODkgMTEwLjY3NCAxMDYuNzY0IDExMC42NzQgMTA3LjI4NyAxMTEuMTk3QzEwNy43NTEgMTExLjY2IDEwNy43NzMgMTExLjg0OSAxMDcuNDE1IDExMi4yODFDMTA3LjE3NiAxMTIuNTY4IDEwNi44NjYgMTEyLjgwMyAxMDYuNzI2IDExMi44MDNDMTA2LjU4NiAxMTIuODAzIDEwNi4yNzcgMTEyLjU2OCAxMDYuMDM4IDExMi4yODFaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDEwNy4yMjRDMTA1LjY4MiAxMDYuNzk1IDEwNS42OTQgMTA2LjYxMiAxMDYuMTA2IDEwNi4yMDFDMTA2LjY1IDEwNS42NTcgMTA3LjczOCAxMDUuODggMTA3LjczOCAxMDYuNTM2QzEwNy43MzggMTA2Ljk3IDEwNy4wNzIgMTA3Ljc0NyAxMDYuNyAxMDcuNzQ3QzEwNi41NzUgMTA3Ljc0NyAxMDYuMjc3IDEwNy41MTIgMTA2LjAzOCAxMDcuMjI0WiIgZmlsbD0iIzk4QzZEOCIvPgo8cGF0aCBkPSJNMTE3Ljk1NSAxMzYuNTQxQzExNC41NzggMTM2LjExMSAxMTAuMDg3IDEzNC44ODEgMTA5LjQxIDEzNC4yNzRDMTA4LjEyMyAxMzMuMTE5IDEwOC45NDIgMTI3Ljg3OSAxMTAuNDY2IDEyNi41NEMxMTEuMzg3IDEyNS43MzEgMTE1LjAzOSAxMjQuNjYzIDExNy4xODIgMTI1LjE2OUMxMTkuNjQyIDEyNS43NDkgMTIzLjkzMiAxMjguNjQ4IDEyNC4yNSAxMjkuNTA4QzEyNC42MjUgMTMwLjUyMiAxMjQuMDQ4IDEzNC41NDEgMTIzLjMyMyAxMzUuNTM2QzEyMi42OTQgMTM2LjM5OSAxMjAuMzI1IDEzNi44NDMgMTE3Ljk1NSAxMzYuNTQxWk0xMjAuNTQgMTM0LjA4NEMxMjEuMjk3IDEzMy4zOTkgMTIxLjM0MiAxMzEuODQyIDEyMC42MjcgMTMxLjEyNkMxMTkuNDkgMTI5Ljk4OSAxMTcuMTEyIDEzMC44NjkgMTE3LjExMiAxMzIuNDI4QzExNy4xMTIgMTM0LjMwNCAxMTkuMTg5IDEzNS4zMDcgMTIwLjU0IDEzNC4wODRaTTExOC41ODIgMTMzLjEyNUMxMTguMzExIDEzMi40MTkgMTE4LjY4IDEzMS42MDggMTE5LjI3MiAxMzEuNjA4QzEyMCAxMzEuNjA4IDEyMC4yMjggMTMyLjE3MyAxMTkuODEyIDEzMi45NDlDMTE5LjM4MSAxMzMuNzU1IDExOC44NTMgMTMzLjgzIDExOC41ODIgMTMzLjEyNVpNMTE2LjQyMiAxMzMuMTQzQzExNy4wOTQgMTMyLjMzMyAxMTYuNzExIDEzMS4xNzYgMTE1LjY4MSAxMzAuOTAyQzExNC41ODEgMTMwLjYxIDExNC4wNTIgMTMxLjE4MyAxMTQuODY5IDEzMS43OEMxMTUuNjgzIDEzMi4zNzUgMTE1LjU1MSAxMzIuNjc5IDExNC41MzMgMTMyLjU1N0MxMTMuMzI3IDEzMi40MTMgMTEyLjgzMyAxMzEuNTY2IDExMy4yNTQgMTMwLjM2MkMxMTMuNjgxIDEyOS4xNDIgMTE1LjE5NyAxMjguODYzIDExNS43NTMgMTI5LjkwMkMxMTYuMTkzIDEzMC43MjUgMTE2LjYyNyAxMzAuNzkzIDExNi45IDEzMC4wODNDMTE3LjIyOSAxMjkuMjI0IDExNS45MTQgMTI4LjIzNyAxMTQuNDQgMTI4LjIzN0MxMTEuOTkzIDEyOC4yMzcgMTEwLjgxNiAxMzEuMDc0IDExMi41NDYgMTMyLjgwM0MxMTMuNSAxMzMuNzU4IDExNS43NDkgMTMzLjk1NSAxMTYuNDIyIDEzMy4xNDNaIiBmaWxsPSIjN0JBOUI5Ii8+CjxwYXRoIGQ9Ik0xMjAuMTE4IDEzOC41OTJDMTE4Ljc1MSAxMzcuMjk3IDEyMS45MTYgMTM2LjA5NiAxMjMuNjA2IDEzOC4yODNDMTI0LjQ2OSAxMzkuMzIyIDEyMi44NTMgMTM5Ljk3NiAxMjAuMTE4IDEzOC41OTJaTTEwNi4xMiAxMzUuNjU4QzEwNC43NTEgMTM0LjI4OSAxMDYuOTYzIDEzMy45MDQgMTA4LjU4MSAxMzUuMjNMMTA5Ljk0MSAxMzUuOTJMMTA4LjA1OSAxMzYuMDYxQzEwNy4yMTUgMTM2LjA2MiAxMDYuMzQzIDEzNS44ODEgMTA2LjEyIDEzNS42NThWMTM1LjY1OFpNMTI1LjM2MyAxMjcuODY2QzEyNS4wODIgMTI3LjQxIDEyOC4zMzMgMTI2LjI2NyAxMjguNjg4IDEyNi40ODZDMTI4Ljg0NiAxMjYuNTg0IDEyOC45NzUgMTI3LjMzNyAxMjguOTc1IDEyOC4xNjFDMTI4Ljk3NSAxMjkuMjM5IDEyOC44NCAxMjkuNjU4IDEyOC40OSAxMjkuNjU4QzEyOC4yMjIgMTI5LjY1OCAxMjUuNDc2IDEyOC4wNDcgMTI1LjM2MyAxMjcuODY2Wk0xMTguNjQxIDEyMS4wOTVDMTE0Ljg3MSAxMjAuNzUzIDExNS4zNzggMTE5LjMyNyAxMTYuMzYxIDExOC40OTlDMTE2LjkyOCAxMTguMDIxIDExNy4zNCAxMTcuNzc5IDExNy45MTMgMTE3LjY2N0MxMTcuOTEzIDExNy42NjcgMTE3Ljg3MiAxMTkuMDQ2IDExOC42NjYgMTE5LjMxOUMxMTkuMjk3IDExOS41MzYgMTIwLjI3OCAxMTkuMDY0IDEyMC40NzEgMTE4LjY3NUMxMjIuMTg4IDExNS4yMDEgMTIxLjQxNSAxMTguNjk5IDEyMS43OTQgMTE5LjUzMkMxMjMuMjM4IDEyNC4wMDcgMTIwLjE4OCAxMjEuNzMgMTE4LjY0MSAxMjEuMDk1Wk0xMDQuOTEyIDEyMS4yM0MxMDQuMjQyIDEyMC40ODcgMTAzLjY5MyAxMTkuNzI5IDEwMy42OTMgMTE5LjU0NEMxMDMuNjkzIDExOS4xMTYgMTA0Ljc0NiAxMTkuMTEyIDEwNS44NjMgMTE5LjUzN0MxMDYuNTkzIDExOS44MTQgMTEwLjM0NyAxMjAuMDA2IDExMC4zNDcgMTIxLjE1M0MxMTAuMzQ3IDEyMS44OTkgMTA4Ljg5IDEyMy43NjIgMTA4LjcyNiAxMjMuNzYyQzEwOC41NjMgMTIzLjc2MiAxMDUuNTgzIDEyMS45NzIgMTA0LjkxMiAxMjEuMjNaIiBmaWxsPSIjQTZCM0MyIi8+CjxwYXRoIGQ9Ik0xMTguNDYzIDEyMS4wMDhDMTE4LjQ0MiAxMjEuMTQgMTE4LjQxNiAxMjEuMjk5IDExOC4zOTcgMTIxLjQxOUMxMTguMzcgMTIxLjU5MiAxMTguMzQ1IDEyMS43NTUgMTE4LjMyMyAxMjEuOTAzQzExOC4zIDEyMi4wNTEgMTE4LjI4MSAxMjIuMTg1IDExOC4yNjMgMTIyLjMwOEMxMTguMjQ1IDEyMi40MzEgMTE4LjIyOSAxMjIuNTQyIDExOC4yMTYgMTIyLjYzOUMxMTguMjAzIDEyMi43MzYgMTE4LjE5MSAxMjIuODIgMTE4LjE4MyAxMjIuODlDMTE4LjE3NSAxMjIuOTYxIDExOC4xNjggMTIzLjAxOCAxMTguMTY1IDEyMy4wNjFDMTE4LjE2MyAxMjMuMDgzIDExOC4xNjQgMTIzLjEgMTE4LjE2MyAxMjMuMTE1QzExOC4xNjIgMTIzLjEyOSAxMTguMTYyIDEyMy4xNCAxMTguMTYzIDEyMy4xNDhDMTE4LjE2MyAxMjMuMTUyIDExOC4xNjQgMTIzLjE1NiAxMTguMTY1IDEyMy4xNThMMTE4LjE2NyAxMjMuMTZDMTE4LjE2OCAxMjMuMTYgMTE4LjE3IDEyMy4xNTkgMTE4LjE3MSAxMjMuMTU4QzExOC4xNzIgMTIzLjE1NyAxMTguMTczIDEyMy4xNTUgMTE4LjE3NSAxMjMuMTUyQzExOC4xNzggMTIzLjE0NSAxMTguMTgyIDEyMy4xMzUgMTE4LjE4NyAxMjMuMTIxQzExOC4xOTIgMTIzLjEwNyAxMTguMTk2IDEyMy4wODkgMTE4LjIwMiAxMjMuMDY3QzExOC4yMTMgMTIzLjAyNSAxMTguMjI4IDEyMi45NjkgMTE4LjI0NSAxMjIuODk3QzExOC4yNjIgMTIyLjgyNSAxMTguMjgyIDEyMi43MzggMTE4LjMwNSAxMjIuNjM1QzExOC4zMjcgMTIyLjUzMyAxMTguMzUxIDEyMi40MTUgMTE4LjM3OSAxMjIuMjgxQzExOC40MDYgMTIyLjE0OCAxMTguNDM4IDEyMS45OTggMTE4LjQ3MSAxMjEuODMzQzExOC41MTYgMTIxLjYxMyAxMTguNTczIDEyMS4zMjEgMTE4LjYyOCAxMjEuMDQzTDExOC40NjMgMTIxLjAwOFoiIGZpbGw9IiM0QzlDQkIiLz4KPHBhdGggZD0iTTMwLjk5ODYgMTQyLjY3M0MyNi40NDg5IDE0MC45MTYgMjEuNDY2NCAxMzYuODA1IDE4Ljc5MjEgMTMyLjYwM0MxNS40MDUzIDEyNy4yODEgMTQuNDA1MiAxMjMuNDE5IDE1LjA2MzYgMTE4LjIwM0MxNS42ODA2IDExMy4zMTUgMTcuMzU1OCAxMTAuNTU3IDIyLjIyMzIgMTA2LjQxM0MyOS4wNDE3IDEwMC42MDggMzQuNDg0NiA5Ny45Nzk5IDQwLjU5NDcgOTcuNTQxNUM0OS44MjEyIDk2Ljg3OTQgNTUuMDk3OCA5OS44MTA5IDU3LjY0NjUgMTA3LjAxNUM1OC41MDQzIDEwOS40MzkgNTguNjg1NiAxMTUuNTcgNTguMDU0NCAxMjAuODA5QzU3LjgzMTEgMTIyLjY2MyA1Ny40Nzg5IDEyNi4wNTUgNTcuMjcxOCAxMjguMzQ4QzU2LjU0NjcgMTM2LjM3NSA1NC40MTQ3IDE0MC40MTcgNDkuNjI1MyAxNDIuODQzQzQ4LjI5NzkgMTQzLjUxNiA0Ny43MTk5IDE0My41NjEgNDAuNjkyNCAxNDMuNTM5QzMzLjUxNTEgMTQzLjUxNiAzMy4wODEgMTQzLjQ3OCAzMC45OTg2IDE0Mi42NzNaTTAuMDc5NTc2NSAxMDQuOTY1QzAuMDgxNTcyMyAxMDMuODUzIDAuMTQ0NDAzIDEwMy40MzggMC4yMTk0MjcgMTA0LjA0NEMwLjI5NDQ1MSAxMDQuNjUgMC4yOTI4OTkgMTA1LjU2IDAuMjE1OTk2IDEwNi4wNjdDMC4xMzkwNzkgMTA2LjU3MyAwLjA3NzY4MzggMTA2LjA3OCAwLjA3OTU3NjUgMTA0Ljk2NVpNMC4wMDE3MzM1MSAxMDIuMjRDMC4wMTc4OTkyIDEwMS44NDggMC4wOTc3ODk3IDEwMS43NjggMC4yMDUzOTggMTAyLjAzN0MwLjMwMjc3MiAxMDIuMjggMC4yOTA3OTcgMTAyLjU3MSAwLjE3ODc4NyAxMDIuNjgzQzAuMDY2Nzc0OCAxMDIuNzk1IC0wLjAxMjkwMjEgMTAyLjU5NiAwLjAwMTczMzUxIDEwMi4yNFpNMC4wMjgzNDQ0IDc1LjEzMjZDMC4wMjgzNDQ0IDc0LjY2OTEgMC4xMDQ4NTcgNzQuNDc5NSAwLjE5ODM2OSA3NC43MTEyQzAuMjkxODg0IDc0Ljk0MyAwLjI5MTg4NCA3NS4zMjIyIDAuMTk4MzY5IDc1LjU1MzlDMC4xMDQ4NTcgNzUuNzg1NyAwLjAyODM0NDQgNzUuNTk2MSAwLjAyODM0NDQgNzUuMTMyNloiIGZpbGw9InVybCgjcGFpbnQyX3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTM1LjMxMzMgMTEyLjU0NEwzNy45IDExMS4wOTFMMzkuMzUxNyAxMjEuNDZMMzkuMjQ4IDEyOC45MjZMMzUuMzcwMyAxMzAuODUxTDMyLjkyMjkgMTI2LjQzN0wzNS4zMTMzIDExMi41NDRaIiBmaWxsPSIjMzA2MEFEIi8+CjxwYXRoIGQ9Ik0zOC43Mjk1IDExNi42OUwzOS4zNTE2IDEyNC41NzFMNDMuMjkxOSAxMjUuOTE5TDQzLjM5NTYgMTA5LjIyNUw0MC4xODEyIDExMC4wNTRMMzguNzI5NSAxMTYuNjlaIiBmaWxsPSIjNjA2MzY4Ii8+CjxwYXRoIGQ9Ik0zNS4zNjk5IDEzMC44NTFDMzMuMDMyNCAxMjkuNzMgMjkuNzMwMSAxMjYuNDA4IDI4LjY2MjEgMTI0LjEwM0MyOC4wMzMyIDEyMi43NDUgMjcuNjM5NyAxMjIuMzEgMjYuODkyMyAxMjIuMTQ2QzI1Ljc4NzkgMTIxLjkwMyAyNS44MDg0IDEyMS45NzEgMjYuMzIxOCAxMjAuMjU4QzI2LjU5MzYgMTE5LjM1IDI2LjkwMzIgMTE4Ljk1NCAyNy4zNDA2IDExOC45NTRDMjcuNjg2MiAxMTguOTU0IDI5LjE1OTcgMTE3Ljg0MSAzMC42MTQ5IDExNi40OEMzMi4wNzAyIDExNS4xMTkgMzMuNzIyNiAxMTMuNjc3IDM0LjI4NjkgMTEzLjI3NUwzNS4zMTI5IDExMi41NDRMMzUuNjc2NyAxMTQuMjQxQzM3LjA0NDMgMTIwLjYxOCAzNy4wMjcgMTI0LjI2NiAzNS42MTk4IDEyNi4yNDNDMzQuNTUyMSAxMjcuNzQ0IDM0Ljg0MTcgMTI4LjM5MyAzNi41NzkxIDEyOC4zOTNDMzcuMzk3IDEyOC4zOTMgMzcuODIyMiAxMjguMTg1IDM4LjEzMDUgMTI3LjYzNEMzOC42NzQ4IDEyNi42NjMgMzguNjMxNSAxMjAuOTY4IDM4LjA0NTQgMTE2LjQyNkMzNy4zNDQ2IDExMC45OTUgMzcuMzEyNyAxMTEuMTY3IDM5LjE1NzggMTEwLjM5OEM0MC4wMzYgMTEwLjAzMSA0MC44MTggMTA5Ljc5NyA0MC44OTU1IDEwOS44NzdDNDAuOTczMSAxMDkuOTU2IDQwLjgxOTEgMTExLjIzNSA0MC41NTM0IDExMi43MThDMzkuNjA2NyAxMTguMDAzIDM5LjkzMTUgMTIzLjIzOCA0MS4yNzUyIDEyNC4zNTNDNDEuNjg1MSAxMjQuNjkzIDQxLjcyMzMgMTIzLjk2OSA0MS41NDUyIDExOS4yMzJDNDEuMzE1NiAxMTMuMTIzIDQxLjUxNzQgMTEwLjkxOSA0Mi40MDc1IDEwOS44MkM0Mi45MTk4IDEwOS4xODggNDMuMjEzIDEwOS4xMTYgNDQuNDk1OSAxMDkuMzFDNDYuOTU2NiAxMDkuNjgyIDQ2Ljk1MDYgMTA5LjY2OCA0Ni4yMDk5IDExMy4yNTRDNDUuODQ5NiAxMTQuOTk5IDQ1LjQ0ODMgMTE3LjIyNCA0NS4zMTggMTE4LjJMNDUuMDgxMiAxMTkuOTczTDQ1Ljk0NDYgMTE5LjY0NUM0Ni40MTk1IDExOS40NjUgNDcuNDQxMyAxMTguOTk0IDQ4LjIxNTMgMTE4LjU5OUw0OS42MjI2IDExNy44ODFMNDkuMzg5MyAxMTguNjdDNDkuMjYxIDExOS4xMDUgNDkuMDY3MiAxMjAuNzAxIDQ4Ljk1ODcgMTIyLjIxOEw0OC43NjE0IDEyNC45NzZMNDcuMTA5NSAxMjQuMzI1QzQ2LjIwMSAxMjMuOTY3IDQ1LjMxMTYgMTIzLjY3NCA0NS4xMzMxIDEyMy42NzRDNDQuOTQxIDEyMy42NzQgNDQuODA4NSAxMjUuNTMyIDQ0LjgwODUgMTI4LjIyNEM0NC44MDg1IDEzMS44OTEgNDQuNzE3MSAxMzIuNzc1IDQ0LjMzODYgMTMyLjc3NUM0MS4wMTk4IDEzMi4yOTUgMzguNzg0MiAxMzIuNDU4IDM1LjM2OTkgMTMwLjg1MVoiIGZpbGw9IiNFRUMyM0IiLz4KPHBhdGggZD0iTTQxLjE2MzkgMTMyLjQxOEM0MS4xNjM5IDEzMi4yODMgNDEuMzQwOCAxMzAuNTE1IDQxLjY2OTIgMTI4LjE0NUM0MS45OTc3IDEyNS43NzUgNDIuMzU3NyAxMjIuOTYxIDQyLjQ2OTIgMTIxLjg5M0M0Mi41ODA4IDEyMC44MjQgNDIuNzk2OSAxMjAuMDI3IDQyLjk0OTUgMTIwLjEyMUM0My42NDQzIDEyMC41NTEgNDQuMjkzOCAxMjkuMzc1IDQ0LjA0NzQgMTMxLjQyQzQzLjg5MDkgMTMyLjcxOSA0My42NTk1IDEzMi43MzcgNDMuMzAxOSAxMzIuNjc3QzQyLjY2MDIgMTMyLjU3IDQxLjkxMDEgMTMyLjUxMyA0MS4xNjM5IDEzMi40MThaTTMyLjU5NjkgMTIyLjczNUMzMS42ODEzIDEyMi41ODMgMzAuODI0MiAxMjEuNDQyIDMwLjgyNDIgMTIwLjM3NEMzMC44MjQyIDExOS40MDcgMzIuMjM4MSAxMTcuOTM3IDMzLjE2ODUgMTE3LjkzN0MzNC42NjY2IDExNy45MzcgMzUuODM5MiAxMjAuNTEzIDM0Ljk0ODQgMTIxLjg0N0MzNC41NTE3IDEyMi40NDEgMzMuNDE0NyAxMjIuODcgMzIuNTk2OSAxMjIuNzM1WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTExNC4wMjEgMTM3LjQ5OEMxMDcuMzE4IDEzNS43NCAxMDYuMjczIDEzNC4xMDQgMTA2LjYwOSAxMjUuODkxQzEwNi43MTYgMTIzLjI4MSAxMDYuODg1IDEyMC41MzkgMTA2Ljk4NiAxMTkuNzk3QzEwNy41NDUgMTE1LjY2NCAxMDguMzQyIDExMi4xNzMgMTA5LjA4NiAxMTAuNTk5QzEwOS43NDQgMTA5LjIwNiAxMDkuODQyIDEwOC43MTIgMTA5LjUzNSAxMDguMzQyQzEwOC40NSAxMDcuMDM1IDExMC4zNzEgMTA0Ljk5NSAxMTEuOTEgMTA1LjgxOUMxMTIuNDU3IDEwNi4xMTEgMTEyLjkxOCAxMDYuMDc3IDExMy45NDUgMTA1LjY2NUMxMTcuMDYyIDEwNC40MTkgMTIzLjExNyAxMDUuNTcxIDEyNi44MTQgMTA4LjExNUMxMjguMTQ5IDEwOS4wMzMgMTI4LjQ4NSAxMDkuMTM2IDEyOS4wMzggMTA4Ljc5MUMxMzAuNTIzIDEwNy44NjMgMTMyLjQ1MSAxMTAuNTczIDEzMS4yNSAxMTEuOUMxMzAuNzg3IDExMi40MTIgMTMwLjc1NSAxMTIuNzA1IDEzMS4wNjUgMTEzLjU5NkMxMzEuODUgMTE1Ljg0NiAxMzEuNzk0IDExNi4xMDQgMTI4LjAwNiAxMjcuNzE5QzEyNS41OCAxMzUuMTYgMTI0LjU0MyAxMzcuMzY1IDEyMy4yNjQgMTM3LjgxMUMxMjEuNjUzIDEzOC4zNzMgMTE2LjcyMyAxMzguMjA2IDExNC4wMjEgMTM3LjQ5OFpNMTIyLjU0MSAxMzUuMjI5QzEyMy4wMDMgMTM0LjU0MSAxMjMuMzU5IDEzMy4zMTMgMTIzLjUxMyAxMzEuODc4QzEyMy43OTQgMTI5LjI1IDEyMy44MTkgMTI5LjI4MyAxMjAuMzE3IDEyNy42MDFDMTE3LjY4NSAxMjYuMzM4IDExMy44MzMgMTI1Ljk3MiAxMTEuODU1IDEyNi43OThDMTEwLjgyOSAxMjcuMjI3IDExMC41OTUgMTI3LjU1NiAxMTAuMDU5IDEyOS4zMjdDMTA5LjExNiAxMzIuNDQzIDEwOS43MDMgMTM0LjMwOCAxMTEuODMyIDEzNC45NThDMTEzLjEzOSAxMzUuMzU3IDEyMC4wOSAxMzYuNDMgMTIwLjk4IDEzNi4zN0MxMjEuNTA0IDEzNi4zMzUgMTIyLjA4MiAxMzUuOTEyIDEyMi41NDEgMTM1LjIyOVpNMTE5LjQ1MSAxMjIuNDJDMTE5LjYzIDEyMi4wODEgMTE5LjY4NSAxMjEuNTU2IDExOS42ODUgMTIxLjU1NkMxMjAuMTU2IDEyMS42NTQgMTIwLjQ4IDEyMS42ODMgMTIwLjcyNiAxMjEuNzE5QzEyMS4xMDUgMTIxLjc3NCAxMjEuNTY0IDEyMS41ODcgMTIxLjc0NSAxMjEuMzAzQzEyMi4xNDkgMTIwLjY3MSAxMjAuNDc0IDExNy43MDEgMTE5LjU0OCAxMTcuNDA4QzExOC42NjggMTE3LjEyOCAxMTUuOTM1IDExOC44MjcgMTE1LjkzNSAxMTkuNjUzQzExNS45MzUgMTIwLjAyMiAxMTYuNTUzIDEyMC40MTEgMTE2LjgzMSAxMjAuNTE4QzExNi45ODIgMTIwLjU3NiAxMTcuMjUyIDEyMC43MzggMTE3LjU3NiAxMjAuOTM4QzExNy4zOCAxMjEuMzUxIDExNy4zNDYgMTIxLjQ3OCAxMTcuMzEyIDEyMS43MjRDMTE3LjIwNCAxMjIuNDk4IDExNy40MzUgMTIyLjY4NiAxMTguNDE2IDEyMi45MTJDMTE5LjE1MSAxMjIuOTkxIDExOS4yNSAxMjIuODQ1IDExOS40NTEgMTIyLjQyWk0xMjcuMjY1IDExNi44MzZDMTMwLjkzMyAxMTMuNDc1IDEyNy4wNzQgMTA3LjQxIDEyMi43MDUgMTA5LjY3QzEyMS42MjYgMTEwLjIyOCAxMjAuNjU0IDExMi4wMTkgMTIwLjY1NCAxMTMuNDQ5QzEyMC42NTQgMTE1LjY2NCAxMjIuODMxIDExNy45MzcgMTI0Ljk1NyAxMTcuOTQxQzEyNS43MzMgMTE3Ljk0MyAxMjYuNDE0IDExNy42MTcgMTI3LjI2NSAxMTYuODM2Wk0xMTcuMTQxIDExNS42MjJDMTE5LjAzNSAxMTQuNDY4IDExOS44MDQgMTExLjc5NSAxMTguODAyIDEwOS44NTdDMTE4LjA4MiAxMDguNDY0IDExNy4wMDggMTA3LjgzIDExNS4zNjYgMTA3LjgzQzExMS41MDggMTA3LjgzIDEwOS40ODUgMTEzLjIzNCAxMTIuNDY1IDExNS41NzhDMTEzLjU3IDExNi40NDggMTE1Ljc1NCAxMTYuNDY4IDExNy4xNDEgMTE1LjYyMloiIGZpbGw9IiMwOTk2RDEiLz4KPHBhdGggb3BhY2l0eT0iMC45IiBkPSJNMzMuMTc1NyAyNkwzMi4zODEgMjcuMjE5QzMxLjk0NDkgMjcuODg5IDMxLjI4MzYgMjkuMzI0OCAzMC45MDkxIDMwLjQwODhDMzAuNTM0NyAzMS40OTI4IDI5Ljk5MjIgMzIuOTcwMSAyOS43MDQzIDMzLjY5MjhDMjkuNDE2MyAzNC40MTU0IDI5LjE3MzMgMzYuMTE1IDI5LjE2MzggMzcuNDdDMjkuMTQzNiA0MC4zNTMgMjguNzQ5NCA0MS44ODg4IDI3LjYzNjQgNDMuNDIyNUMyNi45MjEyIDQ0LjQwOCAyNi42NzUxIDQ0LjUyMjMgMjUuNDUzMiA0NC40NDFDMjMuODk5NSA0NC4zMzc3IDIyLjgxNTMgNDUuMDgxNSAyMS4yMTcgNDcuMzQ4MUMyMC43MTY1IDQ4LjA1OCAyMC4xMzY3IDQ4LjYzOTMgMTkuOTI4OSA0OC42MzkzQzE5LjcyMTEgNDguNjM5MyAxOS4xODA1IDQ4LjIzMTkgMTguNzI2MiA0Ny43MzUxQzE4LjI3MTkgNDcuMjM4MiAxNy45OTQ0IDQ3LjAxMTUgMTguMTExIDQ3LjIyOThDMTguNDkgNDcuOTM5NiAxNy41OTA0IDUzLjI2MTUgMTYuODM3OCA1NC43NjQyQzE1Ljk2NyA1Ni41MDI4IDE0LjM4NzIgNTcuNzIzMyAxMi41MzU0IDU4LjA4NDNDMTEuMDA1OSA1OC4zODI2IDExLjIyODMgNTguMDkgOC43MTc5NSA2My4wOTQ2QzYuNjYzMzQgNjcuMTkwNyA1Ljk2OTM1IDY3LjgyNjQgMi42NTUzMyA2OC42NTQyQzEuNDk3MTcgNjguOTQzNCAwLjQ1ODc1OSA2OS43MTQyIDAgNzAuNTI4N1Y3Mi44MjYzQzAuMzU2MTkzIDcyLjc0NzMgMC44Mjg3MjggNzIuNjIwMyAxLjYxNDk5IDcyLjM4NzNDMi45OTQ4NiA3MS45NzgyIDQuODIzNDggNzEuNDgwNiA1LjY3ODEgNzEuMjgwNkM3LjcyOTM0IDcwLjgwMDYgOS4zOTM1MSA2OS4yMTk1IDkuOTIyNzkgNjcuMjQ2N0MxMC4yNDI0IDY2LjA1NTQgMTAuNzAzMyA2NS40MjQxIDExLjk5OTIgNjQuNDA1OEMxMy4yODQgNjMuMzk2MiAxMy43NTY1IDYyLjc1NDYgMTQuMDY3MSA2MS41OTY5QzE0LjM3MzkgNjAuNDUzMiAxNC43MDExIDYwLjAwNCAxNS40NDA3IDU5LjcwNjNDMTcuMDE1MyA1OS4wNzI0IDIwLjY0NDQgNTUuODY1NSAyMS4yMzYzIDU0LjU4NThDMjEuNTM3MyA1My45MzQ5IDIyLjExOTggNTIuNTM5OSAyMi41Mjg3IDUxLjQ4NjJDMjMuMjYzNCA0OS41OTMyIDI1LjAxOTUgNDcuMzI2MSAyNS43NTAxIDQ3LjMyNjFDMjUuOTU3NCA0Ny4zMjYxIDI3LjEwOSA0Ni43MjQ4IDI4LjMwOTMgNDUuOTkwOEMzMS4yMzc2IDQ0LjIwMDMgMzEuNTk5MSA0My40MTM2IDMxLjU5OTEgMzguODYxNEMzMS41OTkxIDM1LjY4MTQgMzEuNjg3IDM1LjE1NTcgMzIuMzQ2OCAzNC4zNjg0QzMzLjAwMzcgMzMuNTg0NyAzMy4wOTk3IDMzLjAyNDkgMzMuMTM1MSAyOS43MzkxTDMzLjE3NTcgMjZaTTI0LjMzMzggNjMuNTk1OEMyNC4yMDM3IDYzLjYxMzUgMjQuMDczOCA2My42NjE4IDIzLjkyNTggNjMuNzM2MkMyMi45Mzk3IDY0LjIzMTUgMjIuNjEzNiA2NC45MTAyIDIxLjU5NzMgNjguNTg2QzIxLjAxNzkgNzAuNjgxNSAyMC40MjEyIDcyLjA4MTkgMTkuODc5OCA3Mi42MTc4QzE5LjQyMzUgNzMuMDY5NSAxOC45MzgzIDczLjg4MjMgMTguODAxIDc0LjQyNDJDMTguMTgxIDc2Ljg3MTUgMTYuNzQ1NCA3OC42MDczIDE0LjQ0OTUgNzkuNjg3MUMxMi40NTU1IDgwLjYyNDkgMTAuNjQwNyA4Mi4xMTc5IDkuMTExMDIgODQuMDc5OEM4LjMyMTA5IDg1LjA5MjkgNy40ODc1MiA4Ni4wMjkgNy4yNTY3OCA4Ni4xNjA5QzYuNzYxNjQgODYuNDQ0IDUuNzA4OTkgODguODk0MSA0LjYwNzg1IDkyLjMyOEMzLjcxMDkyIDk1LjEyNSAxLjcxMDY1IDk3LjQ2ODIgMC40Nzg1MTQgOTcuMTY1OEMwLjI1NDkzMSA5Ny4xMTA5IDAuMTA2OTExIDk3LjExMzcgMCA5Ny4yOTAxVjEwNC40OTJDMC41NTY0MTEgMTAzLjY2NCAxLjIzNDIgMTAyLjYyOSAyLjU2MzQ3IDEwMC41NEM0LjM0NTExIDk3LjczOTcgNi40NjM4NSA5NC40ODY3IDcuMjcxNzMgOTMuMzEyNEM4LjA3OTYgOTIuMTM4MSA5LjM1Njg3IDkwLjE0ODggMTAuMTA4NiA4OC44OTE2QzExLjIzNDIgODcuMDA5MiAxMi4xMDE2IDg2LjEyNDMgMTUuMDI4NCA4My44ODMzTDE4LjU4MzEgODEuMTYyN0wyMC40MDUzIDc1Ljk4ODFDMjMuMTg5NSA2OC4wODE3IDIzLjk1ODUgNjcuMTk5NyAyNy43NjI0IDY3LjU1MTVDMjkuNDYwNSA2Ny43MDg1IDMwLjYzMzQgNjcuNjE5NiAzMi4wNzM0IDY3LjIyNjdDMzQuMTgyNiA2Ni42NTEyIDM0LjQ3OTkgNjYuMTg3MiAzMy4zNTUxIDY1LjIzMThDMzIuNjU1NCA2NC42Mzc2IDMxLjMxNTMgNjQuNjQgMjguMjM4OCA2NS4yNDE4QzI3LjMzOTcgNjUuNDE3NyAyNi45MzQzIDY1LjI3NTcgMjUuODkzMiA2NC40MTc4QzI1LjEwNTEgNjMuNzY4NCAyNC43MjQxIDYzLjU0MjYgMjQuMzMzOCA2My41OTU4Wk0zMS43MTY2IDgzLjI1OThDMzEuNjYwNSA4My4yNzQ5IDMxLjU5MjEgODMuMzEyNiAzMS41MTE1IDgzLjM3MjFDMzEuMjcwOSA4My41NSAzMC42NTM2IDgzLjc5IDMwLjEzNzkgODMuOTA1NEMyOS4zODc2IDg0LjA3MzQgMjkuMDQ5OCA4NC40OTMyIDI4LjQ0ODIgODYuMDA0NUMyOC4wMzQ3IDg3LjA0MzIgMjcuNDkxNSA4OC4xMTUzIDI3LjI0MTIgODguMzg2M0MyNi45OTA5IDg4LjY1NzMgMjYuNDc1IDg5LjU1MSAyNi4wOTQxIDkwLjM3MzJDMjQuOTQyMiA5Mi44NTkyIDIzLjUyODIgOTQuMDYxOCAxOS45MDExIDk1LjY0MjFDMTYuNjEwNyA5Ny4wNzU2IDE2LjU2NyA5Ny4xMDg2IDE1LjY2NzEgOTguOTgwMkMxNS4xMDI3IDEwMC4xNTQgMTQuNjc5MiAxMDEuNzU3IDE0LjU0OTkgMTAzLjIwNUMxNC4yNzMxIDEwNi4zMDIgMTMuNTk2MiAxMDcuMDMxIDkuNTIxMTggMTA4LjYxNkM1LjkxNDk3IDExMC4wMTkgMy41NzcwMiAxMTEuNzQ1IDMuNDE1ODMgMTEzLjEyM0MzLjI0MDQxIDExNC42MjMgMi41OTgxMyAxMTUuMDg0IDAuOTI5MjYgMTE0LjkwN0MwLjUzOTUyOSAxMTQuODY2IDAuMjQ0MzMyIDExNC44NTEgMCAxMTQuODgxVjEyMUMxLjAyNjU4IDExOS42NTggMi43MTkyMiAxMTguMzYxIDYuMjI0OTcgMTE2LjIwNkMxMC4zNjA0IDExMy42NjUgMTIuNzA2NSAxMTEuNTc1IDEzLjU3NzkgMTA5LjY1NEMxMy45MjYgMTA4Ljg4NyAxNC42NTQ3IDEwNy4zOTMgMTUuMTk3MSAxMDYuMzM0QzE1LjczOTUgMTA1LjI3NSAxNi4yODQ1IDEwMy45MDEgMTYuNDA4NCAxMDMuMjgxQzE2LjczMjQgMTAxLjY2IDE4LjMwMzIgMTAwLjM0OSAyMC42NjU5IDk5LjcyNkMyNS4xNzcgOTguNTM3MiAyNS43NDYxIDk4LjAxNjMgMjYuMzUyNSA5NC41MzU0QzI2Ljg1NjEgOTEuNjQ1MiAyOC42MTIgODguMDY2MSAzMC41MjY3IDg2LjAyNDZDMzEuMzA5NSA4NS4xOSAzMS45NDk1IDg0LjE4IDMxLjk0OTUgODMuNzc5MUMzMS45NDk1IDgzLjM3NjYgMzEuODg0OSA4My4yMTQ2IDMxLjcxNjYgODMuMjU5OFoiIGZpbGw9IiMzNTkxMzYiLz4KPHBhdGggZD0iTTguMDgxNzcgNzQuNDA4M0M2LjAzNDE5IDczLjg3MDUgNS44OTQ0MyA3My43MzAyIDYuMDQyNzEgNzIuMzYxNUM2LjEzMDI1IDcxLjU1MzMgNi4yNDAxNyA3MC44NTc1IDYuMjg2OTYgNzAuODE1NEM2LjMzMzc2IDcwLjc3MzMgNy43MDQ0MyA3MS4wNjg3IDkuMzMyOTIgNzEuNDcxOUMxMS45NzY4IDcyLjEyNjQgMTIuNDg2NiA3Mi4xMjM4IDE0LjA5NDcgNzEuNDQ3NEMxNi40MDQxIDcwLjQ3NiAxNy4yMzgzIDY4Ljc3NTMgMTYuNDAyMiA2Ni43NDMxQzE1Ljk0MjQgNjUuNjI1NyAxNC45MjA0IDY0LjgyODEgMTIuMjM4NCA2My40OTRDNy42NDE3IDYxLjIwNzUgNi41MzA2MiA1OS45OTkgNi4yNzQ2NSA1Ny4wMDc2QzUuODkzMzcgNTIuNTUxOSA4Ljg0Njg5IDQ5Ljk4MzYgMTQuMzUyMiA0OS45ODM2QzE1Ljk3MzMgNDkuOTgzNiAxNy44NDg3IDUwLjE5MzcgMTguNTE5NiA1MC40NTA1QzE5LjYzNjggNTAuODc4MSAxOS42OTM3IDUxLjAzOTkgMTkuMTk0NSA1Mi4zNzEyQzE4LjcxODIgNTMuNjQxNiAxOC41MDI3IDUzLjc2ODggMTcuNDg2MSA1My4zNzk2QzE1LjUxOTkgNTIuNjI3IDEyLjE5NDggNTIuODU1NCAxMS4wMDU4IDUzLjgyNDdDOS43OTA1OCA1NC44MTUzIDkuNTU3MzYgNTYuOTI1MiAxMC41MzI0IDU4LjEwNzlDMTAuODcyNyA1OC41MjA3IDEyLjkwODcgNTkuNzc5IDE1LjA1NjkgNjAuOTA0MUMxOC40NjI4IDYyLjY4ODEgMTkuMDgyIDYzLjIyMTQgMTkuODk1MSA2NS4wNzE5QzIxLjExNTIgNjcuODQ4NyAyMC42Nzg4IDcwLjM0MzcgMTguNjA3OCA3Mi40MzIzQzE2LjE2MDcgNzQuOTAwMiAxMi41NDEzIDc1LjU3OTYgOC4wODE3NyA3NC40MDgzWk0yOC45MzI4IDc0LjM4MjNDMjUuMjM2NSA3Mi45OTU4IDIzLjcxMzggNzAuNTQwNSAyMy43MTM4IDY1Ljk2NjdDMjMuNzEzOCA2Mi41MDM0IDI0LjU1ODggNjAuNDY5NyAyNi44MzMzIDU4LjQ1OTNDMjguMzk2NCA1Ny4wNzc2IDI4Ljk4NjcgNTYuODY4NiAzMS4zMjY4IDU2Ljg2ODZDMzIuODE2OCA1Ni44Njg2IDM0LjY4MjEgNTcuMjEyOSAzNS41MDA1IDU3LjYzODlDMzcuMzE2OSA1OC41ODQ2IDM4LjgzMTkgNjEuNzQyNiAzOC44NDY2IDY0LjYxNDNMMzguODU3MyA2Ni43MDQ0SDMyLjk0MjNIMjcuMDI3M0wyNy4zNDIgNjguMDU2OUMyNy41MTUgNjguODAwNyAyOC4zMzU0IDcwLjAzOTEgMjkuMTY1IDcwLjgwOUMzMC41NzczIDcyLjExOTYgMzAuODk0OSA3Mi4xOTc5IDM0LjE1NDcgNzIuMDM4NUMzNy4zOTM4IDcxLjg4MDEgMzcuNjQ2MyA3MS45NDA5IDM3Ljc4NCA3Mi45MTIyQzM3Ljk2MjYgNzQuMTcyNSAzNy4xODgyIDc0LjUyMzYgMzMuNTI0MiA3NC44NDM0QzMxLjgwODUgNzQuOTkzMiAzMC4xMDU2IDc0LjgyMjIgMjguOTMyOCA3NC4zODIzWk0zNS4wODgxIDYyLjUyNDJDMzQuOTU1OCA2MC42NzMgMzMuMzY0NCA1OS4zMjc2IDMxLjMwNzIgNTkuMzI3NkMyOS43MDg2IDU5LjMyNzYgMjguNzI4MSA2MC4wOTA5IDI3LjY2MiA2Mi4xNjU1QzI2Ljc0NTcgNjMuOTQ4NiAyNy40MTIgNjQuMjk2NSAzMS40NDg4IDY0LjE0MjVMMzUuMTkzNiA2My45OTk2TDM1LjA4ODEgNjIuNTI0MlpNNDQuMzkxNCA3NC4zNzA4QzQyLjU3MTEgNzMuNTQ2OCA0MS43MDU5IDcyLjEyODYgNDEuNjQyMSA2OS44NjQ3QzQxLjU0MTYgNjYuMjk0NCA0NC4xMTEgNjQuMTQ2OCA0OS4yNTc2IDYzLjQ5OTRDNTEuODQxNCA2My4xNzQ0IDUyLjA0OTUgNjMuMDU3NyA1MS43NTkzIDYyLjA5NjlDNTAuOTY1IDU5LjQ2NjYgNDguMjQyMyA1OC41NjQ0IDQ0LjkyNzYgNTkuODMzMUM0My40MzAzIDYwLjQwNjMgNDMuNDg2OCA2MC40MzEzIDQzLjA1MTQgNTkuMDAzMkM0Mi43ODAxIDU4LjExMzUgNDIuOTk4MiA1Ny44NTExIDQ0LjM5NSA1Ny4zODdDNDYuOTM2NCA1Ni41NDI3IDUxLjM3NDUgNTYuNjM0MiA1Mi43Nzc5IDU3LjU2QzU0LjkzOTIgNTguOTg1NyA1NS4zNzUxIDYwLjYwMTkgNTUuNTQ4NyA2Ny44MzI4TDU1LjcxMDYgNzQuNTc0NUw1NC4xMjMgNzQuNTc0QzUzLjAzNzcgNzQuNTczMyA1Mi41MzUzIDc0LjMzOTYgNTIuNTM1MyA3My44MzU0QzUyLjUzNTMgNzIuOTA2NiA1Mi4wNTI4IDcyLjkwMjQgNTAuNzUyIDczLjgxOTZDNDkuMjAxMyA3NC45MTMgNDYuMTY5NyA3NS4xNzU3IDQ0LjM5MTQgNzQuMzcwOFpNNTAuOTQ3NyA3MC44MTc1QzUxLjcwOTkgNjkuOTczNSA1Mi4wNDY4IDY4Ljk4NiA1Mi4wNDY4IDY3LjU5NjZWNjUuNTkyOUw0OS44MTgyIDY1Ljg0NThDNDUuNzE5IDY2LjMxMSA0My44NzQ5IDY5LjUxMTggNDYuNTA1OCA3MS41OTUzQzQ4LjAzNzEgNzIuODA3OSA0OS4zNTk0IDcyLjU3NjMgNTAuOTQ3NyA3MC44MTc1Wk05Mi4wMDIgNzQuNTc4OEM4OC40MDI0IDczLjQ4NjQgODYuMjQxOSA3MC4yNDY2IDg2LjI0MTkgNjUuOTQxQzg2LjI0MTkgNjAuNDA0MiA4OS41MjExIDU2LjgzMjUgOTQuNjQxIDU2Ljc5MjVDOTguNjk4OSA1Ni43NjA5IDEwMS4yNjMgNTkuNDY3OSAxMDEuNTAzIDY0LjAzNjZMMTAxLjYzIDY2LjQ1ODVMOTYuMjU2MiA2Ni41ODg1QzkzLjMwMDcgNjYuNjYgOTAuNjE1OCA2Ni44MTYgOTAuMjg5NyA2Ni45MzUxQzg5LjM4MTggNjcuMjY2OCA5MC40Nzg4IDY5Ljk0NyA5Mi4wMzA2IDcxLjE4ODZDOTMuMTU5OSA3Mi4wOTIxIDkzLjc3MSA3Mi4xOTEyIDk2LjgxNzggNzEuOTY1QzEwMC4xOTMgNzEuNzE0NCAxMDAuMzE4IDcxLjc0NDkgMTAwLjUzMSA3Mi44Njg0QzEwMC43MyA3My45MTMzIDEwMC41MDkgNzQuMDgzNSA5OC4zNTc5IDc0LjU0OEM5NS42MTAzIDc1LjE0MTIgOTMuODgyOSA3NS4xNDk2IDkyLjAwMiA3NC41Nzg4Wk05Ny45NjU5IDYzLjA4MjNDOTcuOTY1OSA2MS4xMjMxIDk2LjE4NTggNTkuMzI3NiA5NC4yNDM0IDU5LjMyNzZDOTIuMjQ5NCA1OS4zMjc2IDkwLjE0OTkgNjEuMjU2MiA5MC4xNDk5IDYzLjA4NzhDOTAuMTQ5OSA2NC4yMDcgOTAuMjc5NyA2NC4yNDU1IDk0LjA1NzkgNjQuMjQ1NUM5Ny44NDE0IDY0LjI0NTUgOTcuOTY1OSA2NC4yMDg0IDk3Ljk2NTkgNjMuMDgyM1pNMTEwLjA0MiA3NC40N0MxMDYuMTQ5IDczLjQzIDEwMy45MyA2OS40MDI5IDEwNC41NCA2NC40ODYyQzEwNS4xNjYgNTkuNDQ0MiAxMDcuOTUxIDU2Ljg3MzYgMTEyLjc5MiA1Ni44NzA1QzExNS4zNzQgNTYuODY5IDExNS44MDUgNTcuMDI0OSAxMTcuMjYgNTguNDlDMTE4Ljg3OCA2MC4xMTg1IDExOS45NDEgNjIuODM0MyAxMTkuOTQ2IDY1LjM1MkwxMTkuOTQ5IDY2LjcwNDRIMTE0LjA4N0MxMDguNDg4IDY2LjcwNDQgMTA4LjIyNSA2Ni43NTAxIDEwOC4yMjUgNjcuNzIwM0MxMDguMjI1IDcwLjcxMjIgMTEwLjk1MiA3Mi4zNDQyIDExNS4zNTcgNzEuOTg4MkMxMTguMTM2IDcxLjc2MzcgMTE4LjQyMyA3MS44MzE1IDExOC42NjcgNzIuNzY3NEMxMTguOTk4IDc0LjA0NDMgMTE4Ljk2OCA3NC4wNzIgMTE2LjY5IDc0LjYwNDNDMTE0LjM2NyA3NS4xNDcgMTEyLjQzIDc1LjEwNzggMTEwLjA0MiA3NC40N1pNMTE2LjExOSA2Mi43NzAxQzExNS44NjQgNjAuODc4NyAxMTQuMTU5IDU5LjMyNzYgMTEyLjMzNSA1OS4zMjc2QzExMC41NjYgNTkuMzI3NiAxMDguMjI0IDYxLjYwMTQgMTA4LjIyNCA2My4zMTk1QzEwOC4yMjQgNjQuMjEyOCAxMDguNTI3IDY0LjI3NDYgMTEyLjI1NSA2NC4xNDI0QzExNi4yNDEgNjQuMDAxMSAxMTYuMjgzIDYzLjk4NjIgMTE2LjExOSA2Mi43NzAxWk0xMjYuODk0IDc0LjI0N0MxMjQuODI2IDczLjIyMDggMTIzLjUzMiA3MS4zNTg5IDEyMi44OTQgNjguNDg4OUMxMjEuOTc1IDY0LjM2MDUgMTIzLjA3NCA2MC40OTE0IDEyNS44MyA1OC4xNTY3QzEyNy44MjcgNTYuNDY1NyAxMzEuODM4IDU2LjMzMDUgMTMzLjgwNSA1Ny44ODc5TDEzNS4wOTIgNTguOTA3MlY1My45NTM2VjQ5SDEzNy4wNDZIMTM5VjYxLjc4NjVWNzQuNTczMUgxMzcuMjlDMTM1Ljc2MiA3NC41NzMxIDEzNS41OCA3NC40MzMxIDEzNS41OCA3My4yNTc5VjcxLjk0MjdMMTM0LjQ4MSA3Mi45NzkyQzEzMi4yMzMgNzUuMDk5OCAxMjkuNTI1IDc1LjU1MjMgMTI2Ljg5NCA3NC4yNDdaTTEzMy42NjQgNzAuNjc2NkMxMzQuOTc2IDY5LjM1NTcgMTM1LjA5MiA2OC45NTM3IDEzNS4wOTIgNjUuNzIwOUMxMzUuMDkyIDYyLjQ1NzIgMTM0Ljk4NSA2Mi4wOTQ1IDEzMy42MDcgNjAuNzA3NkMxMzIuMzY1IDU5LjQ1NzUgMTMxLjg2NSA1OS4yNjIyIDEzMC41NTQgNTkuNTE1NEMxMjYuMTQ0IDYwLjM2NzIgMTI0LjgzNSA2OC4xNDIzIDEyOC41OTggNzEuMTIyOUMxMzAuMzQ4IDcyLjUwODEgMTMxLjk4OCA3Mi4zNjM1IDEzMy42NjQgNzAuNjc2NlpNNjEuNDEwNyA2Ni4wNTg0QzYwLjAyMiA2MS4zNzU0IDU4Ljg4NTkgNTcuMzkxOSA1OC44ODU5IDU3LjIwNjJDNTguODg1OSA1Ny4wMjA2IDU5LjcyNzQgNTYuODY4NiA2MC43NTU5IDU2Ljg2ODZINjIuNjI1OUw2NC4yNTUxIDYzLjg3NjZDNjUuMTUxMiA2Ny43MzEgNjUuOTU2IDcwLjY2MzMgNjYuMDQzNyA3MC4zOTI5QzY2LjEzMTMgNzAuMTIyNCA2Ny4wNTU3IDY3LjAyNDEgNjguMDk3OCA2My41MDc4TDY5Ljk5MjYgNTcuMTE0NUg3MS41OTY5SDczLjIwMTJMNzQuODg2NSA2My4yNjE5Qzc1LjgxMzQgNjYuNjQzIDc2LjY4MzIgNjkuNzQxMiA3Ni44MTk0IDcwLjE0N0M3Ni45NTU3IDcwLjU1MjcgNzcuODQxOSA2Ny43MzEgNzguNzg5IDYzLjg3NjZMODAuNTEwOSA1Ni44Njg2SDgyLjQ1MTdDODMuOTUxNyA1Ni44Njg2IDg0LjMzMDMgNTcuMDMxNiA4NC4xMTkgNTcuNTg2QzgzLjk2ODYgNTcuOTgwNiA4Mi42OTU2IDYxLjk2NDEgODEuMjkwMiA2Ni40MzgyTDc4LjczNDkgNzQuNTczMUg3Ni45NjM1SDc1LjE5MjFMNzMuNTgxMiA2OC43OTQ1QzcyLjY5NTIgNjUuNjE2MyA3MS44NTg1IDYyLjY4NDEgNzEuNzIxNyA2Mi4yNzgzQzcxLjQ0NjUgNjEuNDYxNiA3MS41OTExIDYxLjA3MzMgNjkuMTExNCA2OS4yODYzTDY3LjUxNTIgNzQuNTczMUg2NS43MjU0SDYzLjkzNTZMNjEuNDEwNyA2Ni4wNTg0WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTE5Ljk3NDYgMTE5LjI3NEMxOS4wMTU4IDExOC4xMTkgMjAuMzM4MSAxMTYuMTY2IDIxLjU2MDQgMTE2LjkzMkMyMi43NjMyIDExNy42ODYgMjIuMzMwNSAxMTkuNzYxIDIwLjk3MDQgMTE5Ljc2MUMyMC42NDUgMTE5Ljc2MSAyMC4xOTY5IDExOS41NDIgMTkuOTc0NiAxMTkuMjc0WiIgZmlsbD0idXJsKCNwYWludDNfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMjMuODM2IDExMi42M0MyMi4xMTUyIDExMS42ODkgMjEuODAyOSAxMDkuNzk3IDIzLjE0NjUgMTA4LjQ1M0MyNC4wODU3IDEwNy41MTQgMjUuMjEgMTA3LjM5NCAyNi4yNTA3IDEwOC4xMjNDMjcuODU3MSAxMDkuMjQ5IDI3LjQ2NTUgMTEyLjAxMyAyNS42MDc5IDExMi42NkMyNS42MDc5IDExMi42NiAyNC40NzQ1IDExMi45NzMgMjMuODM2IDExMi42M1oiIGZpbGw9InVybCgjcGFpbnQ0X3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTMzLjc4NTUgMTIwLjAzNkMzMy43ODU1IDEyMC4xODggMzMuNzU1NyAxMjAuMzM4IDMzLjY5NzYgMTIwLjQ3OUMzMy42Mzk1IDEyMC42MTkgMzMuNTU0NCAxMjAuNzQ2IDMzLjQ0NzEgMTIwLjg1M0MzMy4zMzk4IDEyMC45NjEgMzMuMjEyNCAxMjEuMDQ2IDMzLjA3MjMgMTIxLjEwNEMzMi45MzIxIDEyMS4xNjIgMzIuNzgxOCAxMjEuMTkyIDMyLjYzMDEgMTIxLjE5MkMzMi40NzgzIDEyMS4xOTIgMzIuMzI4MSAxMjEuMTYyIDMyLjE4NzkgMTIxLjEwNEMzMi4wNDc3IDEyMS4wNDYgMzEuOTIwMyAxMjAuOTYxIDMxLjgxMyAxMjAuODUzQzMxLjcwNTcgMTIwLjc0NiAzMS42MjA2IDEyMC42MTkgMzEuNTYyNiAxMjAuNDc5QzMxLjUwNDUgMTIwLjMzOCAzMS40NzQ2IDEyMC4xODggMzEuNDc0NiAxMjAuMDM2QzMxLjQ3NDYgMTE5LjczIDMxLjU5NjMgMTE5LjQzNiAzMS44MTMgMTE5LjIxOUMzMi4wMjk3IDExOS4wMDMgMzIuMzIzNiAxMTguODgxIDMyLjYzMDEgMTE4Ljg4MUMzMi45MzY1IDExOC44ODEgMzMuMjMwNCAxMTkuMDAzIDMzLjQ0NzEgMTE5LjIxOUMzMy42NjM4IDExOS40MzYgMzMuNzg1NSAxMTkuNzMgMzMuNzg1NSAxMjAuMDM2WiIgZmlsbD0iYmxhY2siLz4KPHBhdGggZD0iTTEyMC43MjEgMTE4LjI3NkMxMjAuNzIxIDExOC42MjMgMTIwLjU2NSAxMTguOTU2IDEyMC4yODcgMTE5LjIwMUMxMjAuMDA5IDExOS40NDYgMTE5LjYzMiAxMTkuNTg0IDExOS4yMzkgMTE5LjU4NEMxMTguODQ2IDExOS41ODQgMTE4LjQ2OSAxMTkuNDQ2IDExOC4xOTEgMTE5LjIwMUMxMTcuOTEzIDExOC45NTYgMTE3Ljc1NyAxMTguNjIzIDExNy43NTcgMTE4LjI3NkMxMTcuNzU3IDExOC4xMDUgMTE3Ljc5NSAxMTcuOTM1IDExNy44NyAxMTcuNzc2QzExNy45NDQgMTE3LjYxNyAxMTguMDUzIDExNy40NzMgMTE4LjE5MSAxMTcuMzUyQzExOC4zMjkgMTE3LjIzIDExOC40OTIgMTE3LjEzNCAxMTguNjcyIDExNy4wNjhDMTE4Ljg1MSAxMTcuMDAzIDExOS4wNDQgMTE2Ljk2OSAxMTkuMjM5IDExNi45NjlDMTE5LjQzMyAxMTYuOTY5IDExOS42MjYgMTE3LjAwMyAxMTkuODA2IDExNy4wNjhDMTE5Ljk4NiAxMTcuMTM0IDEyMC4xNDkgMTE3LjIzIDEyMC4yODcgMTE3LjM1MkMxMjAuNDI0IDExNy40NzMgMTIwLjUzMyAxMTcuNjE3IDEyMC42MDggMTE3Ljc3NkMxMjAuNjgyIDExNy45MzUgMTIwLjcyMSAxMTguMTA1IDEyMC43MjEgMTE4LjI3NloiIGZpbGw9IiM1OTY4NkYiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82NzlfMTkxMCIgeDE9IjExIiB5MT0iLTQuNzc3NjhlLTA3IiB4Mj0iMTY0LjUiIHkyPSIxNTAuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNjRCN0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzAwNURBRCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50MV9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMTI0LjkwNCAxMjguNjM2KSByb3RhdGUoLTAuMDQ4NTUyKSBzY2FsZSgzNi4yODAyIDMxLjQwMzIpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50Ml9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMzkuMzUxMiAxMjEuNDYpIHJvdGF0ZSgtMC4wNTczKSBzY2FsZSg1MC4yMjc2IDUxLjMwOTEpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50M19yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjAuOTU0OCAxMTguMjYpIHNjYWxlKDEuMzA5MjkgMS41MDEyNykiPgo8c3RvcCBzdG9wLWNvbG9yPSIjODhBM0QwIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzVEODNCRiIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50NF9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjQuNzg0MyAxMTAuMjIxKSBzY2FsZSgyLjQ3Mjc3IDIuNTY5NTQpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzg4QTNEMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM1RDgzQkYiLz4KPC9yYWRpYWxHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"], ["spec", "lifecycle"], ["spec", "lifecycle", "enabled"], ["spec", "lifecycle", "storageClass"]] secrets: exclude: [] include: [] diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index 3436e5f0f..54a9c0338 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -61,6 +61,12 @@ update: patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch + # Backport of seaweedfs/seaweedfs#11157, released in 4.46: drop the patch + # once the vendored chart carries [grpc.s3]. --forward makes that point + # fail here, where BSD patch would otherwise reverse it and remove the + # section without an error. + patch --forward --no-backup-if-mismatch -p4 < patches/s3-grpc-tls.patch + patch --forward --no-backup-if-mismatch -p4 < patches/storage-class-string.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml $(MAKE) update-image diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml index ad112981f..1ce6f3cab 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml @@ -330,7 +330,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.admin.data.storageClass }} + {{- with .Values.admin.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.admin.data.size }} @@ -344,7 +346,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.admin.logs.storageClass }} + {{- with .Values.admin.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.admin.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml index 4a402f318..f5329a24e 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml @@ -19,7 +19,7 @@ spec: accessModes: {{- toYaml (.Values.allInOne.data.accessModes | default (list "ReadWriteOnce")) | nindent 4 }} {{- if .Values.allInOne.data.storageClass }} - storageClassName: {{ .Values.allInOne.data.storageClass }} + storageClassName: {{ .Values.allInOne.data.storageClass | quote }} {{- end }} resources: requests: diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml index e505faf1c..ced2d597a 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -451,7 +451,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.filer.data.storageClass }} + {{- with .Values.filer.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.filer.data.size }} @@ -467,7 +469,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.filer.logs.storageClass }} + {{- with .Values.filer.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.filer.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml index 914f0dfd1..5df83e687 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml @@ -336,7 +336,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.master.data.storageClass }} + {{- with .Values.master.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.master.data.size }} @@ -352,7 +354,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.master.logs.storageClass }} + {{- with .Values.master.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.master.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml index a067e1de9..17b82cca0 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml @@ -97,6 +97,19 @@ data: cert = "/usr/local/share/ca-certificates/worker/tls.crt" key = "/usr/local/share/ca-certificates/worker/tls.key" + {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000). + Without this section, LoadServerTLS("grpc.s3") returns nil and the S3 + server serves its internal gRPC (IAM cache propagation, s3 lifecycle + delete RPCs) as plaintext, while peers dial it with mTLS credentials: + "tls: first record does not look like a TLS handshake". + Always uses the internal CA-signed client certificate (already mounted + on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that + secret is for the public HTTPS listener and may be issued by a public + CA which internal gRPC peers (trusting only grpc.ca) would reject. */}} + [grpc.s3] + cert = "/usr/local/share/ca-certificates/client/tls.crt" + key = "/usr/local/share/ca-certificates/client/tls.key" + # use this for any place needs a grpc client # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" [grpc.client] diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml index 04011d1d6..88b13822d 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml @@ -404,7 +404,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $dir.storageClass }} + {{- with $dir.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $dir.size }} @@ -422,7 +424,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $volume.idx.storageClass }} + {{- with $volume.idx.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $volume.idx.size }} @@ -438,7 +442,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $volume.logs.storageClass }} + {{- with $volume.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $volume.logs.size }} diff --git a/packages/system/seaweedfs/patches/s3-grpc-tls.patch b/packages/system/seaweedfs/patches/s3-grpc-tls.patch new file mode 100644 index 000000000..d4d45fbd0 --- /dev/null +++ b/packages/system/seaweedfs/patches/s3-grpc-tls.patch @@ -0,0 +1,24 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +index a067e1de9..17b82cca0 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +@@ -97,6 +97,19 @@ data: + cert = "/usr/local/share/ca-certificates/worker/tls.crt" + key = "/usr/local/share/ca-certificates/worker/tls.key" + ++ {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000). ++ Without this section, LoadServerTLS("grpc.s3") returns nil and the S3 ++ server serves its internal gRPC (IAM cache propagation, s3 lifecycle ++ delete RPCs) as plaintext, while peers dial it with mTLS credentials: ++ "tls: first record does not look like a TLS handshake". ++ Always uses the internal CA-signed client certificate (already mounted ++ on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that ++ secret is for the public HTTPS listener and may be issued by a public ++ CA which internal gRPC peers (trusting only grpc.ca) would reject. */}} ++ [grpc.s3] ++ cert = "/usr/local/share/ca-certificates/client/tls.crt" ++ key = "/usr/local/share/ca-certificates/client/tls.key" ++ + # use this for any place needs a grpc client + # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" + [grpc.client] diff --git a/packages/system/seaweedfs/patches/storage-class-string.patch b/packages/system/seaweedfs/patches/storage-class-string.patch new file mode 100644 index 000000000..0c4562105 --- /dev/null +++ b/packages/system/seaweedfs/patches/storage-class-string.patch @@ -0,0 +1,128 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +index ad112981f..1ce6f3cab 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +@@ -330,7 +330,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.admin.data.storageClass }} ++ {{- with .Values.admin.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.admin.data.size }} +@@ -344,7 +346,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.admin.logs.storageClass }} ++ {{- with .Values.admin.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.admin.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +index 4a402f318..f5329a24e 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +@@ -19,7 +19,7 @@ spec: + accessModes: + {{- toYaml (.Values.allInOne.data.accessModes | default (list "ReadWriteOnce")) | nindent 4 }} + {{- if .Values.allInOne.data.storageClass }} +- storageClassName: {{ .Values.allInOne.data.storageClass }} ++ storageClassName: {{ .Values.allInOne.data.storageClass | quote }} + {{- end }} + resources: + requests: +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +index e505faf1c..ced2d597a 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +@@ -451,7 +451,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.filer.data.storageClass }} ++ {{- with .Values.filer.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.filer.data.size }} +@@ -467,7 +469,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.filer.logs.storageClass }} ++ {{- with .Values.filer.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.filer.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +index 914f0dfd1..5df83e687 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +@@ -336,7 +336,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.master.data.storageClass }} ++ {{- with .Values.master.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.master.data.size }} +@@ -352,7 +354,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.master.logs.storageClass }} ++ {{- with .Values.master.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.master.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +index 04011d1d6..88b13822d 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +@@ -404,7 +404,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $dir.storageClass }} ++ {{- with $dir.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $dir.size }} +@@ -422,7 +424,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $volume.idx.storageClass }} ++ {{- with $volume.idx.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $volume.idx.size }} +@@ -438,7 +442,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $volume.logs.storageClass }} ++ {{- with $volume.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $volume.logs.size }} diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml index 7840d200d..d43aa176e 100644 --- a/packages/system/seaweedfs/tests/cosi_image_test.yaml +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -7,7 +7,7 @@ suite: seaweedfs COSI driver image pin # BucketAccess and ignores the accessPolicy parameter. The chart ships a # -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on # v0.1.2 a credential meant to be read-only is silently granted read-write. -# The package values.yaml overrides cosi.image to v0.3.1, which honours +# The package values.yaml overrides cosi.image to v0.4.0, which honours # accessPolicy: readonly. # # This test does not set the image itself; it relies on the package values.yaml @@ -25,7 +25,7 @@ release: namespace: cozy-seaweedfs tests: - - it: pins the cosi-driver container to v0.3.1 (read-only access enforcement) + - it: pins the cosi-driver container to v0.4.0 (read-only access enforcement) template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: seaweedfs.cosi.enabled: true @@ -35,4 +35,4 @@ tests: value: seaweedfs-cosi-driver - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml new file mode 100644 index 000000000..7f3cec8ff --- /dev/null +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -0,0 +1,175 @@ +suite: seaweedfs runs the S3 lifecycle worker + +# SeaweedFS stores a bucket lifecycle configuration on PUT and acts on it only +# from the s3_lifecycle job of the plugin worker, which the admin server +# schedules (weed/worker/tasks/s3_lifecycle/handler.go). Without an admin and a +# worker in the release, a rule is accepted and nothing ever expires. + +templates: + - charts/seaweedfs/templates/admin/admin-statefulset.yaml + - charts/seaweedfs/templates/worker/worker-deployment.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: runs the admin server that schedules lifecycle jobs + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: &enabled + seaweedfs: + admin: + enabled: true + secret: + existingSecret: seaweedfs-admin-credentials + worker: + enabled: true + asserts: + - isKind: + of: StatefulSet + - equal: + path: metadata.name + value: seaweedfs-admin + + # The admin keeps its job state and any change an operator makes in its UI + # in -dataDir; on an emptyDir a restart would silently undo them. + - it: keeps the admin data directory on a persistent volume + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-dataDir=/data' + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: admin-data + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 1Gi + + - it: runs the worker against the admin of the same release + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - isKind: + of: Deployment + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + + # The worker executes every job type it registers for. Lifecycle is the only + # one this release asks for: vacuum, balance and erasure coding move data and + # are not something enabling lifecycle should start doing as a side effect. + - it: registers the worker for the s3_lifecycle job only + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-jobType=s3_lifecycle ' + + - it: runs no Lance worker container, since the Lance catalog is off + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + + - it: gives the admin and the worker explicit resources + set: *enabled + asserts: + - exists: + path: spec.template.spec.containers[0].resources.requests.memory + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - exists: + path: spec.template.spec.containers[0].resources.limits.memory + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 64Mi + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # The admin serves an unauthenticated UI unless a password is set, and a + # tenant namespace admits ingress from the whole cluster. The credentials come + # from a Secret the parent release keeps; the values here only wire them in. + - it: reads the admin credentials from the named Secret + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: WEED_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: seaweedfs-admin-credentials + key: adminPassword + - contains: + path: spec.template.spec.containers[0].env + content: + name: WEED_ADMIN_USER + valueFrom: + secretKeyRef: + name: seaweedfs-admin-credentials + key: adminUser + + - it: renders neither the admin nor the worker unless the parent turns them on + asserts: + - hasDocuments: + count: 0 + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - hasDocuments: + count: 0 + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # Every other component here logs to stderr; the upstream default for these + # two writes to a file in an emptyDir that kubectl logs never shows. + - it: logs the admin and the worker to stderr + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-logtostderr=true' + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-logtostderr=true' + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # One worker replica rolls out by surging a second pod. The subchart's hard + # hostname anti-affinity keeps that pod off the only node of a single-node + # cluster, so every later change to the worker template would hang the + # release on its rollout. + - it: lets the worker roll out on a single node + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - notExists: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution + + # The parent hands the class on as a string, and a class name may be all + # digits; emitted unquoted it turns back into a number the claim rejects. + - it: keeps a numeric admin storage class name a string + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs: + admin: + enabled: true + secret: + existingSecret: seaweedfs-admin-credentials + data: + storageClass: "123" + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + + # An empty storageClassName disables dynamic provisioning; with no class + # named, the claim must leave the field out so the cluster default applies. + - it: leaves the admin claim on the cluster default when no class is named + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName diff --git a/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml new file mode 100644 index 000000000..afac836ed --- /dev/null +++ b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml @@ -0,0 +1,32 @@ +suite: seaweedfs s3 gRPC listener speaks mTLS + +templates: + - charts/seaweedfs/templates/shared/security-configmap.yaml + - charts/seaweedfs/templates/s3/s3-deployment.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + # weed s3 serves its internal gRPC (the LifecycleDelete RPC among it) with + # the [grpc.s3] identity and as plaintext without one, while the worker dials + # it with [grpc.worker] mTLS credentials, so every delete failed the TLS + # handshake. patches/s3-grpc-tls.patch backports the section from upstream. + - it: gives the s3 gRPC server a TLS identity signed by the internal CA + template: charts/seaweedfs/templates/shared/security-configmap.yaml + asserts: + - matchRegex: + path: data["security.toml"] + pattern: '\[grpc\.s3\]\s+cert = "/usr/local/share/ca-certificates/client/tls\.crt"\s+key = "/usr/local/share/ca-certificates/client/tls\.key"' + + # security.toml reaches the s3 pods through a subPath mount, which a running + # container never re-reads, so an upgrade that only changes the ConfigMap + # leaves s3 serving plaintext gRPC until something restarts it. The pod + # template has to change in the same release. + - it: rolls the s3 pods so they load the new section + template: charts/seaweedfs/templates/s3/s3-deployment.yaml + asserts: + - equal: + path: spec.template.metadata.annotations["seaweedfs.cozystack.io/grpc-s3-tls"] + value: "true" diff --git a/packages/system/seaweedfs/tests/storage_class_string_test.yaml b/packages/system/seaweedfs/tests/storage_class_string_test.yaml new file mode 100644 index 000000000..038cf1fed --- /dev/null +++ b/packages/system/seaweedfs/tests/storage_class_string_test.yaml @@ -0,0 +1,123 @@ +suite: seaweedfs claims keep the storage class a string + +# A StorageClass name may be all digits. Emitted unquoted it renders as a +# number, which the claim rejects; and an empty storageClassName disables +# dynamic provisioning, so with no class named the field has to stay out for +# the cluster default to apply. + +templates: + - charts/seaweedfs/templates/master/master-statefulset.yaml + - charts/seaweedfs/templates/filer/filer-statefulset.yaml + - charts/seaweedfs/templates/volume/volume-statefulset.yaml + - charts/seaweedfs/templates/admin/admin-statefulset.yaml + - charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml + # filer-statefulset includes it for its checksum annotation. + - charts/seaweedfs/templates/s3/s3-secret.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: keeps numeric master data and logs storage class names strings + template: charts/seaweedfs/templates/master/master-statefulset.yaml + set: + seaweedfs.master.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.master.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[?(@.metadata.name == "data-tenant-root")].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[?(@.metadata.name == "seaweedfs-master-log-volume")].spec.storageClassName + value: "456" + + - it: leaves the master claims on the cluster default when no class is named + template: charts/seaweedfs/templates/master/master-statefulset.yaml + set: + seaweedfs.master.data: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.master.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps numeric filer data and logs storage class names strings + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + set: + seaweedfs.filer.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.filer.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the filer claims on the cluster default when no class is named + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + set: + seaweedfs.filer.data: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.filer.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps numeric volume idx and logs storage class names strings + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + set: + seaweedfs.volume.dataDirs: [] + seaweedfs.volume.idx: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.volume.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the volume idx and logs claims on the cluster default when no class is named + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + set: + seaweedfs.volume.dataDirs: [] + seaweedfs.volume.idx: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.volume.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps a numeric admin logs storage class name a string + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs.admin.enabled: true + seaweedfs.admin.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the admin logs claim on the cluster default when no class is named + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs.admin.enabled: true + seaweedfs.admin.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps a numeric all-in-one storage class name a string + template: charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml + set: + seaweedfs.allInOne.enabled: true + seaweedfs.allInOne.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + asserts: + - equal: + path: spec.storageClassName + value: "123" diff --git a/packages/system/seaweedfs/tests/volume_storage_class_test.yaml b/packages/system/seaweedfs/tests/volume_storage_class_test.yaml new file mode 100644 index 000000000..037dc0969 --- /dev/null +++ b/packages/system/seaweedfs/tests/volume_storage_class_test.yaml @@ -0,0 +1,42 @@ +suite: seaweedfs volume claims keep the storage class a string + +templates: + - charts/seaweedfs/templates/volume/volume-statefulset.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + # The parent hands volume.storageClass on as a string, and a class name may + # be all digits; emitted unquoted it turns back into a number the claim + # rejects. + - it: keeps a numeric volume storage class name a string + set: + seaweedfs: + volume: + dataDirs: + - name: data1 + type: persistentVolumeClaim + size: 10Gi + storageClass: "123" + maxVolumes: 0 + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + + # An empty storageClassName disables dynamic provisioning; with no class + # named, the claim must leave the field out so the cluster default applies. + - it: leaves the volume claim on the cluster default when no class is named + set: + seaweedfs: + volume: + dataDirs: + - name: data1 + type: persistentVolumeClaim + size: 10Gi + maxVolumes: 0 + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index bce78c47f..64e2562ba 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -207,6 +207,14 @@ seaweedfs: # weed skip startLanceServer and drops the container port and Service port # with it. lancePort: 0 + # patches/s3-grpc-tls.patch adds [grpc.s3] to security.toml, which the s3 + # pods mount by subPath and so never re-read. This annotation changes the + # pod template in the release that brings the section, so an upgrade + # restarts s3 onto it instead of leaving its gRPC in plaintext. The + # upstream s3 template carries no checksum of that ConfigMap, and adding + # one would be another vendored patch for a change made once. + podAnnotations: + seaweedfs.cozystack.io/grpc-s3-tls: "true" # Keep S3 traffic on same-zone endpoints. Native chart support landed in 4.12 # (replaces the former s3-traffic-distribution.patch); the chart gates the # field on Kubernetes >=1.31 and auto-converts to PreferSameZone on >=1.35. @@ -303,7 +311,10 @@ seaweedfs: # S3 actions for every BucketAccess and ignores the accessPolicy parameter. # v0.3.x honours accessPolicy: readonly, so a -readonly BucketAccessClass # actually issues read-only (Read,List) credentials instead of read-write. - image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1" + # v0.4.0 also writes each grant to the per-identity IAM file: v0.3.1 + # rewrites only the legacy identity.json, which the S3 gateways migrate + # concurrently, so a grant could be lost for good. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" @@ -311,6 +322,62 @@ seaweedfs: region: "" sidecar: image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1" + # A bucket lifecycle rule is stored on PUT and acted on only by the + # s3_lifecycle job of the plugin worker, scheduled by the admin server once a + # day (weed/worker/tasks/s3_lifecycle/handler.go at 4.45). The parent chart + # sends both enabled flags from its lifecycle.enabled, together with the + # admin password below; off here, so a render without the parent never + # starts an admin without one. + admin: + enabled: false + # Without a password weed admin serves its UI unauthenticated, and a tenant + # namespace admits ingress from the whole cluster. The parent chart names + # the Secret it keeps; these are the keys it writes. + secret: + userKey: adminUser + pwKey: adminPassword + # Job state and anything changed in the admin UI live in -dataDir. + data: + type: "persistentVolumeClaim" + size: "1Gi" + logs: + type: "" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + worker: + enabled: false + # Vacuum, balance and erasure coding move data; enabling lifecycle must + # not start them. + jobType: "s3_lifecycle" + # The subchart's hard hostname anti-affinity keeps the surge pod of a + # rollout off the node the running worker holds, so on a single-node + # cluster any change to the worker template never finishes rolling out. + # Only required -> preferred, as for the master above. + affinity: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker + topologyKey: kubernetes.io/hostname + logs: + type: "" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "1" + memory: "512Mi" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e91a334a1..dc533ee59 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -12,7 +12,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bootOrder":{"description":"Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.","type":"integer","minimum":1},"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: diff --git a/pkg/apis/apps/defaulting/defaulting.go b/pkg/apis/apps/defaulting/defaulting.go new file mode 100644 index 000000000..f361a7bfd --- /dev/null +++ b/pkg/apis/apps/defaulting/defaulting.go @@ -0,0 +1,201 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package defaulting applies an ApplicationDefinition's structural-schema +// defaults to application values. It is shared by the aggregated apiserver, +// which defaults on the read path, and by anything that has to reason about +// stored values, which carry only the keys a tenant actually wrote. +package defaulting + +import ( + "encoding/json" + "fmt" + "strings" + + internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/runtime" +) + +// BuildSchema parses an OpenAPI-v3 JSON schema string and returns the +// structural-schema form used for defaulting. Returns (nil, nil) when raw +// is empty after trimming. Returns (nil, error) for malformed JSON, +// v1→internal conversion failures, or structural-schema construction +// errors — callers can decide whether to log-and-continue or fail hard. +func BuildSchema(raw string) (*structuralschema.Structural, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + var v1js apiextv1.JSONSchemaProps + if err := json.Unmarshal([]byte(raw), &v1js); err != nil { + return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) + } + scheme := runtime.NewScheme() + _ = internalapiext.AddToScheme(scheme) + _ = apiextv1.AddToScheme(scheme) + var ijs internalapiext.JSONSchemaProps + if err := scheme.Convert(&v1js, &ijs, nil); err != nil { + return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) + } + s, err := structuralschema.NewStructural(&ijs) + if err != nil { + return nil, fmt.Errorf("build structural schema: %w", err) + } + return s, nil +} + +// Apply defaults root in place against s the way the Kubernetes apiserver +// defaults a custom resource. A nil schema leaves root untouched. +func Apply(root *map[string]any, s *structuralschema.Structural) error { + if s == nil { + return nil + } + v := any(*root) + nv, err := applyDefaults(v, s, true) + if err != nil { + return err + } + obj, ok := nv.(map[string]any) + if !ok && nv != nil { + return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) + } + if obj == nil { + obj = map[string]any{} + } + *root = obj + return nil +} + +func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { + if s == nil { + return v, nil + } + + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + default: + // scalar + } + } + + switch effType { + case "object": + mv, isMap := v.(map[string]any) + if !isMap || mv == nil { + if s.Default.Object != nil && !top { + if dm, ok := s.Default.Object.(map[string]any); ok { + mv = cloneMap(dm) + } + } + if mv == nil { + mv = map[string]any{} + } + } + + for name, ps := range s.Properties { + if _, ok := mv[name]; !ok { + if ps.Default.Object != nil { + mv[name] = clone(ps.Default.Object) + } + } + if cur, ok := mv[name]; ok { + cv, err := applyDefaults(cur, &ps, false) + if err != nil { + return nil, err + } + mv[name] = cv + } + } + + if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { + ap := s.AdditionalProperties.Structural + for k, cur := range mv { + if _, isKnown := s.Properties[k]; isKnown { + continue + } + cv, err := applyDefaults(cur, ap, false) + if err != nil { + return nil, err + } + mv[k] = cv + } + } + return mv, nil + + case "array": + sl, isSlice := v.([]any) + if !isSlice || sl == nil { + if s.Default.Object != nil { + if ds, ok := s.Default.Object.([]any); ok { + sl = cloneSlice(ds) + } + } + if sl == nil { + sl = []any{} + } + } + if s.Items != nil { + for i := range sl { + cv, err := applyDefaults(sl[i], s.Items, false) + if err != nil { + return nil, err + } + sl[i] = cv + } + } + return sl, nil + + default: + if v == nil && s.Default.Object != nil { + return clone(s.Default.Object), nil + } + return v, nil + } +} + +func clone(x any) any { + switch t := x.(type) { + case map[string]any: + return cloneMap(t) + case []any: + return cloneSlice(t) + default: + return t + } +} + +func cloneMap(m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = clone(v) + } + return out +} + +func cloneSlice(s []any) []any { + out := make([]any, len(s)) + for i := range s { + out[i] = clone(s[i]) + } + return out +} diff --git a/pkg/apis/apps/defaulting/defaulting_test.go b/pkg/apis/apps/defaulting/defaulting_test.go new file mode 100644 index 000000000..0ce580e56 --- /dev/null +++ b/pkg/apis/apps/defaulting/defaulting_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaulting + +import "testing" + +// A nil map[string]any in an interface is not == nil, so it has to be +// recognised as empty explicitly, at the root and below it, or writing the +// first default into it panics. +func TestApplyDefaultsTypedNilMaps(t *testing.T) { + s, err := BuildSchema(`{"type":"object","properties":{ + "replicas":{"type":"integer","default":2}, + "roles":{"type":"array","default":["worker"],"items":{"type":"string"}}, + "resources":{"type":"object","default":{},"properties":{"preset":{"type":"string","default":"t1.micro"}}}}}`) + if err != nil { + t.Fatal(err) + } + + var root map[string]any + if err := Apply(&root, s); err != nil { + t.Fatal(err) + } + if root["replicas"] != int64(2) { + t.Errorf("root: replicas = %#v, want 2", root["replicas"]) + } + + var nested map[string]any + var roles []any + root = map[string]any{"resources": nested, "roles": roles} + if err := Apply(&root, s); err != nil { + t.Fatal(err) + } + resources, _ := root["resources"].(map[string]any) + if resources["preset"] != "t1.micro" { + t.Errorf("nested: resources = %#v, want the preset default", root["resources"]) + } + if got, _ := root["roles"].([]any); len(got) != 1 || got[0] != "worker" { + t.Errorf("nested: roles = %#v, want the array default", root["roles"]) + } +} diff --git a/pkg/registry/apps/application/quota.go b/pkg/registry/apps/application/quota.go index eb6f20622..5376139c0 100644 --- a/pkg/registry/apps/application/quota.go +++ b/pkg/registry/apps/application/quota.go @@ -233,7 +233,7 @@ func (r *REST) validateTenantResourceQuotas(ctx context.Context, app *appsv1alph } parentLimit := parentQuota[res] allocated := siblings[res] // zero value when no sibling bounds this resource - used := poolUsed[renderedLimitKey(res)] + used := poolUsed[usageKey(res)] remaining := parentLimit.DeepCopy() remaining.Sub(allocated) remaining.Sub(used) @@ -285,25 +285,39 @@ var rawQuotaKeys = map[string]struct{}{ "resourcequotas": {}, } -// renderedLimitKey maps a shorthand resourceQuotas key (the vocabulary admission -// works in) to the rendered ResourceQuota key that carries its limit-side -// budget, mirroring cozy-lib.resources.flatten. The mapping is allocation-ratio -// independent: ratios only scale the requests.* side that flatten derives from -// the same shorthand value, so the limit-side key always holds the operator's -// declared amount. This lets admission read a tenant's current usage from its -// rendered ResourceQuota.status.used without replicating the chart's ratio math. +// limitCountedKeys are the shorthand keys whose usage the ResourceQuota pod +// evaluator reports under limits.*. For every other resource it counts only +// requests.*, so limits. or limits.hugepages-* never carries +// usage even though the chart renders it. +var limitCountedKeys = map[string]struct{}{ + "cpu": {}, + "memory": {}, + "ephemeral-storage": {}, +} + +// usageKey maps a shorthand resourceQuotas key (the vocabulary admission works +// in) to the rendered ResourceQuota key whose status.used carries its usage. +// cozy-lib.resources.flatten renders a shorthand key both as limits. and as +// requests.; usage is read from whichever of the two the quota controller +// counts. For cpu, memory and ephemeral-storage that is the limit side, which +// holds the operator's declared amount whatever allocation ratio scales the +// request side. Every other resource is counted only on the request side, where +// flatten applies no ratio, so requests. holds the declared amount too. +// This lets admission read a tenant's current usage from its rendered +// ResourceQuota.status.used without replicating the chart's ratio math. // -// cpu, memory, ephemeral-storage, custom resources -> "limits." +// cpu, memory, ephemeral-storage -> "limits." // storage -> "requests.storage" (no hard limits.storage) // pods, services, services.loadbalancers, ... -> "" (verbatim) -func renderedLimitKey(raw string) string { - if raw == "storage" { - return "requests.storage" - } +// extended resources, hugepages-, ... -> "requests." +func usageKey(raw string) string { if _, ok := rawQuotaKeys[raw]; ok { return raw } - return "limits." + raw + if _, ok := limitCountedKeys[raw]; ok { + return "limits." + raw + } + return "requests." + raw } // parentNamespaceOf returns the namespace owned by the parent of the tenant that @@ -341,7 +355,7 @@ func (r *REST) poolRootOf(ns string, bounded map[string]bool) string { // created/updated). Usage is read from each member's ResourceQuota.status.used, // taking a per-resource max within a namespace to avoid double-counting the // chart-rendered and controller-allocated quotas, and is keyed in the rendered -// quota key space (see renderedLimitKey). Boundedness is decided by declared +// quota key space (see usageKey). Boundedness is decided by declared // resourceQuotas so the result does not depend on whether Flux has rendered a // freshly-created tenant's quota yet. func (r *REST) parentPoolUsage(ctx context.Context, parentNS, excludeNS string) (map[string]resource.Quantity, error) { diff --git a/pkg/registry/apps/application/quota_test.go b/pkg/registry/apps/application/quota_test.go index f6e5d3054..3ea2b7a2b 100644 --- a/pkg/registry/apps/application/quota_test.go +++ b/pkg/registry/apps/application/quota_test.go @@ -136,19 +136,20 @@ func usageQuota(t *testing.T, namespace, name string, used map[string]string) *c } } -func TestRenderedLimitKey(t *testing.T) { +func TestUsageKey(t *testing.T) { cases := map[string]string{ "cpu": "limits.cpu", "memory": "limits.memory", "ephemeral-storage": "limits.ephemeral-storage", - "devices.com/nvidia": "limits.devices.com/nvidia", + "devices.com/nvidia": "requests.devices.com/nvidia", + "hugepages-2Mi": "requests.hugepages-2Mi", "storage": "requests.storage", "pods": "pods", "services.loadbalancers": "services.loadbalancers", } for raw, want := range cases { - if got := renderedLimitKey(raw); got != want { - t.Errorf("renderedLimitKey(%q) = %q, want %q", raw, got, want) + if got := usageKey(raw); got != want { + t.Errorf("usageKey(%q) = %q, want %q", raw, got, want) } } } @@ -339,6 +340,26 @@ func TestValidateTenantResourceQuotas_PoolUsage(t *testing.T) { } }) + t.Run("parent usage of an extended resource shrinks remaining", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"}) + used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "2"}) + r := newTenantREST(t, parent, used) + + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) == 0 { + t.Fatalf("child example.com/device=1 should be denied when the parent pool already uses 2 of 2") + } + }) + + t.Run("parent usage of an extended resource leaves the rest grantable", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"}) + used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "1"}) + r := newTenantREST(t, parent, used) + + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) > 0 { + t.Fatalf("child example.com/device=1 should be allowed when the parent pool uses 1 of 2, got: %v", errs) + } + }) + // No rendered usage yet (freshly-created parent, Flux has not rendered the // quota): usage degrades to zero and behaviour matches the pre-usage gate. t.Run("absent usage degrades to budget-only check", func(t *testing.T) { diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 5f618b4e3..aa0e24a62 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fluxshard "github.com/cozystack/cozystack/internal/fluxshardoperator" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" "github.com/cozystack/cozystack/pkg/apis/apps/presets" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/apis/apps/validation" @@ -57,7 +58,6 @@ import ( "github.com/cozystack/cozystack/pkg/registry" fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields" "github.com/cozystack/cozystack/pkg/registry/sorting" - internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -110,26 +110,7 @@ type REST struct { // errors — callers can decide whether to log-and-continue (NewREST does) // or fail hard (tests do). func buildSpecSchema(raw string) (*structuralschema.Structural, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - var v1js apiextv1.JSONSchemaProps - if err := json.Unmarshal([]byte(raw), &v1js); err != nil { - return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) - } - scheme := runtime.NewScheme() - _ = internalapiext.AddToScheme(scheme) - _ = apiextv1.AddToScheme(scheme) - var ijs internalapiext.JSONSchemaProps - if err := scheme.Convert(&v1js, &ijs, nil); err != nil { - return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) - } - s, err := structuralschema.NewStructural(&ijs) - if err != nil { - return nil, fmt.Errorf("build structural schema: %w", err) - } - return s, nil + return defaulting.BuildSchema(raw) } // NewREST creates a new REST storage for Application with specific configuration diff --git a/pkg/registry/apps/application/rest_defaulting.go b/pkg/registry/apps/application/rest_defaulting.go index 36518af69..4fe7409dd 100644 --- a/pkg/registry/apps/application/rest_defaulting.go +++ b/pkg/registry/apps/application/rest_defaulting.go @@ -18,8 +18,8 @@ package application import ( "encoding/json" - "fmt" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -51,137 +51,5 @@ func (r *REST) applySpecDefaults(app *appsv1alpha1.Application) error { } func defaultLikeKubernetes(root *map[string]any, s *structuralschema.Structural) error { - v := any(*root) - nv, err := applyDefaults(v, s, true) - if err != nil { - return err - } - obj, ok := nv.(map[string]any) - if !ok && nv != nil { - return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) - } - if obj == nil { - obj = map[string]any{} - } - *root = obj - return nil -} - -func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { - if s == nil { - return v, nil - } - - effType := s.Type - if effType == "" { - switch { - case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): - effType = "object" - case s.Items != nil: - effType = "array" - default: - // scalar - } - } - - switch effType { - case "object": - mv, isMap := v.(map[string]any) - if !isMap || v == nil { - if s.Default.Object != nil && !top { - if dm, ok := s.Default.Object.(map[string]any); ok { - mv = cloneMap(dm) - } - } - if mv == nil { - mv = map[string]any{} - } - } - - for name, ps := range s.Properties { - if _, ok := mv[name]; !ok { - if ps.Default.Object != nil { - mv[name] = clone(ps.Default.Object) - } - } - if cur, ok := mv[name]; ok { - cv, err := applyDefaults(cur, &ps, false) - if err != nil { - return nil, err - } - mv[name] = cv - } - } - - if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { - ap := s.AdditionalProperties.Structural - for k, cur := range mv { - if _, isKnown := s.Properties[k]; isKnown { - continue - } - cv, err := applyDefaults(cur, ap, false) - if err != nil { - return nil, err - } - mv[k] = cv - } - } - return mv, nil - - case "array": - sl, isSlice := v.([]any) - if !isSlice || v == nil { - if s.Default.Object != nil { - if ds, ok := s.Default.Object.([]any); ok { - sl = cloneSlice(ds) - } - } - if sl == nil { - sl = []any{} - } - } - if s.Items != nil { - for i := range sl { - cv, err := applyDefaults(sl[i], s.Items, false) - if err != nil { - return nil, err - } - sl[i] = cv - } - } - return sl, nil - - default: - if v == nil && s.Default.Object != nil { - return clone(s.Default.Object), nil - } - return v, nil - } -} - -func clone(x any) any { - switch t := x.(type) { - case map[string]any: - return cloneMap(t) - case []any: - return cloneSlice(t) - default: - return t - } -} - -func cloneMap(m map[string]any) map[string]any { - out := make(map[string]any, len(m)) - for k, v := range m { - out[k] = clone(v) - } - return out -} - -func cloneSlice(s []any) []any { - out := make([]any, len(s)) - for i := range s { - out[i] = clone(s[i]) - } - return out + return defaulting.Apply(root, s) } diff --git a/pkg/reservation/aggregate.go b/pkg/reservation/aggregate.go new file mode 100644 index 000000000..635c41284 --- /dev/null +++ b/pkg/reservation/aggregate.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import "fmt" + +// Application is one stored application as the aggregator sees it: the +// labelled HelmRelease the aggregated apiserver persisted, reduced to what +// identifies its definition and its undefaulted values. +type Application struct { + Namespace string + Name string + Kind string + Group string + Values []byte +} + +// EvaluatorLookup returns the evaluator of a kind, found by its kind and +// group so that an out-of-tree kind resolves through the same path as an +// in-tree one. +type EvaluatorLookup func(kind, group string) (*Evaluator, error) + +// Failure is an application whose reservation could not be computed. +type Failure struct { + Application Application + Err error +} + +func (f Failure) Error() string { + return fmt.Sprintf("%s %s/%s: %v", f.Application.Kind, f.Application.Namespace, f.Application.Name, f.Err) +} + +// Aggregate sums the reservations of apps per namespace. +// +// An application that cannot be evaluated is returned as a Failure and is +// left out of the sums. The sums are therefore a lower bound whenever a +// failure is returned, and the caller must not treat them as complete: a +// stored application that stops evaluating has to keep its last known +// reservation rather than hand its share of the pool to the next writer. +func Aggregate(apps []Application, lookup EvaluatorLookup, env Environment) (map[string]ResourceList, []Failure) { + perNamespace := map[string]ResourceList{} + var failures []Failure + for _, app := range apps { + ev, err := lookup(app.Kind, app.Group) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + rl, err := ev.EvaluateJSON(app.Values, env) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + sum, ok := perNamespace[app.Namespace] + if !ok { + sum = ResourceList{} + perNamespace[app.Namespace] = sum + } + sum.Add(rl) + } + return perNamespace, failures +} diff --git a/pkg/reservation/environment.go b/pkg/reservation/environment.go new file mode 100644 index 000000000..f6727c867 --- /dev/null +++ b/pkg/reservation/environment.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ErrNotFound is wrapped by an Environment when a named instance type or +// preset does not exist. The evaluator fails closed on it rather than +// reserving nothing. +var ErrNotFound = errors.New("not found") + +// Environment resolves the cluster objects a declaration may reference by +// name. Implementations must be safe for concurrent use and return a list +// the caller may keep: the evaluator does not copy it. +type Environment interface { + Instancetype(name string) (ResourceList, error) + Preset(name string) (ResourceList, error) +} + +// StaticEnvironment is an Environment over fixed tables. +type StaticEnvironment struct { + Instancetypes map[string]ResourceList + Presets map[string]ResourceList +} + +var _ Environment = (*StaticEnvironment)(nil) + +// NewStaticEnvironment classifies every VirtualMachineClusterInstancetype in +// instancetypes and pairs the result with presets. One unclassifiable type +// fails the whole environment, so that a broken catalog is noticed when it is +// loaded rather than when a tenant first orders that type. +func NewStaticEnvironment(instancetypes []map[string]any, presets map[string]ResourceList) (*StaticEnvironment, error) { + env := &StaticEnvironment{ + Instancetypes: make(map[string]ResourceList, len(instancetypes)), + Presets: presets, + } + for _, obj := range instancetypes { + rl, err := ClassifyInstancetype(obj) + if err != nil { + return nil, err + } + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + if name == "" { + return nil, fmt.Errorf("instance type without metadata.name") + } + env.Instancetypes[name] = rl + } + return env, nil +} + +func (e *StaticEnvironment) Instancetype(name string) (ResourceList, error) { + rl, ok := e.Instancetypes[name] + if !ok { + return nil, fmt.Errorf("instance type %q: %w", name, ErrNotFound) + } + return rl, nil +} + +func (e *StaticEnvironment) Preset(name string) (ResourceList, error) { + rl, ok := e.Presets[name] + if !ok { + return nil, fmt.Errorf("resource preset %q: %w", name, ErrNotFound) + } + return rl, nil +} diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go new file mode 100644 index 000000000..6927b0bf2 --- /dev/null +++ b/pkg/reservation/evaluator.go @@ -0,0 +1,453 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + "math" + "regexp" + "slices" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/apiserver/pkg/cel/library" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// ErrNoDeclaration is returned for a kind whose definition declares neither +// its consumption nor an exemption. Such a kind is refused rather than +// treated as free: for an out-of-tree catalog an omitted declaration would +// otherwise be a quota-escalation vector no in-tree test can reach. +var ErrNoDeclaration = errors.New("the application definition declares neither its consumption nor an exemption") + +const ( + valuesVar = "values" + instancetypeFunc = "instanceType" + presetFunc = "preset" + + // costLimit bounds one expression. Declarations are platform-authored, + // but they run on the admission path, so a runaway comprehension must + // fail rather than stall a write. + costLimit = 1_000_000 +) + +var bindingName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// Evaluator computes the reservation of one kind's applications. It is built +// once per declaration and is safe for concurrent use. +type Evaluator struct { + exempt bool + schema *structuralschema.Structural + env *cel.Env + lets []compiled + resources []compiled +} + +type compiled struct { + name string + ast *cel.Ast +} + +// Compile checks decl and prepares it for evaluation. schema is the kind's +// structural OpenAPI schema; stored values are defaulted against it before +// any expression reads them, because the aggregated apiserver persists a +// tenant's spec without its defaults. A nil decl yields ErrNoDeclaration. +func Compile(decl *Declaration, schema *structuralschema.Structural) (*Evaluator, error) { + if decl == nil { + return nil, ErrNoDeclaration + } + if decl.Exempt { + if len(decl.Let) > 0 || len(decl.Resources) > 0 { + return nil, fmt.Errorf("an exempt declaration must not carry let or resources") + } + return &Evaluator{exempt: true}, nil + } + if len(decl.Resources) == 0 { + return nil, ErrNoDeclaration + } + + opts := []cel.EnvOption{ + cel.Variable(valuesVar, cel.DynType), + library.Quantity(), + cel.Function(instancetypeFunc, + cel.Overload("instanceType_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function(presetFunc, + cel.Overload("preset_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function("quantity", + cel.Overload("quantity_int", []*cel.Type{cel.IntType}, apiservercel.QuantityType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + return apiservercel.Quantity{Quantity: resource.NewQuantity(int64(v.(types.Int)), resource.DecimalSI)} + }))), + // The standard _*_ is a singleton dispatching on the left operand's + // traits, so it cannot take a quantity overload; scaling is a method. + cel.Function("mul", + cel.MemberOverload("quantity_mul_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, + cel.BinaryBinding(mulQuantity))), + } + seen := map[string]bool{valuesVar: true} + for _, b := range decl.Let { + if !bindingName.MatchString(b.Name) { + return nil, fmt.Errorf("let %q: not a valid identifier", b.Name) + } + if seen[b.Name] { + return nil, fmt.Errorf("let %q: name already bound", b.Name) + } + seen[b.Name] = true + opts = append(opts, cel.Variable(b.Name, cel.DynType)) + } + env, err := cel.NewEnv(opts...) + if err != nil { + return nil, fmt.Errorf("build CEL environment: %w", err) + } + + ev := &Evaluator{schema: schema, env: env} + for _, b := range decl.Let { + ast, iss := env.Compile(b.Expression) + if iss.Err() != nil { + return nil, fmt.Errorf("let %q: %w", b.Name, iss.Err()) + } + ev.lets = append(ev.lets, compiled{name: b.Name, ast: ast}) + } + for _, key := range slices.Sorted(maps.Keys(decl.Resources)) { + if key == "" { + return nil, fmt.Errorf("resources: empty quota key") + } + ast, iss := env.Compile(decl.Resources[key]) + if iss.Err() != nil { + return nil, fmt.Errorf("resources[%q]: %w", key, iss.Err()) + } + ev.resources = append(ev.resources, compiled{name: key, ast: ast}) + } + return ev, nil +} + +// Exempt reports whether the kind declared that it consumes nothing. +func (e *Evaluator) Exempt() bool { return e.exempt } + +// EvaluateJSON evaluates the reservation of an application from its values +// as stored, a JSON object. +func (e *Evaluator) EvaluateJSON(raw []byte, env Environment) (ResourceList, error) { + values := map[string]any{} + if len(bytes.TrimSpace(raw)) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("decode values: %w", err) + } + m, ok := v.(map[string]any) + if !ok && v != nil { + return nil, fmt.Errorf("decode values: expected an object, got %T", v) + } + if m != nil { + values = m + } + } + return e.Evaluate(values, env) +} + +// Evaluate returns the reservation of an application with the given values. +// The values are defaulted first and are not modified. Anything that cannot +// be resolved — an unknown instance type, a value of the wrong type, a key an +// expression reads without guarding it with has() — is an error, never a +// smaller figure. Keys that evaluate to zero are omitted. +func (e *Evaluator) Evaluate(values map[string]any, env Environment) (ResourceList, error) { + if e.exempt { + return ResourceList{}, nil + } + if env == nil { + return nil, fmt.Errorf("no environment to resolve instance types and presets against") + } + + defaulted := normalizeNumbers(values).(map[string]any) + // Defaulting replaces a value of the wrong type with the schema's + // default, which would turn a malformed spec into a smaller one. + if err := checkTypes(defaulted, e.schema, ""); err != nil { + return nil, err + } + if err := defaulting.Apply(&defaulted, e.schema); err != nil { + return nil, fmt.Errorf("default values: %w", err) + } + + // The environment differs per call, so instanceType and preset are + // declared with a late binding and bound here, per program; cel.Functions + // is deprecated for declaring functions but is still how cel-go supplies + // a late binding. + bindings := cel.Functions( + &functions.Overload{Operator: "instanceType_string", Unary: lookup(env.Instancetype, instancetypeZeroKeys)}, + &functions.Overload{Operator: "preset_string", Unary: lookup(env.Preset, nil)}, + ) + activation := map[string]any{valuesVar: defaulted} + + for _, c := range e.lets { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("let %q: %w", c.name, err) + } + activation[c.name] = out + } + + result := ResourceList{} + for _, c := range e.resources { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + q, err := toQuantity(out) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + if q.Sign() < 0 { + return nil, fmt.Errorf("resources[%q]: negative reservation %s", c.name, q.String()) + } + if q.IsZero() { + continue + } + result[c.name] = q + } + return result, nil +} + +func (e *Evaluator) run(c compiled, bindings cel.ProgramOption, activation map[string]any) (ref.Val, error) { + prg, err := e.env.Program(c.ast, bindings, cel.CostLimit(costLimit)) + if err != nil { + return nil, err + } + out, _, err := prg.Eval(activation) + if err != nil { + return nil, err + } + return out, nil +} + +// instancetypeZeroKeys are present, at zero, in every instanceType() result, +// so that a declaration can read any class of a type without guarding it. +var instancetypeZeroKeys = []string{ + KeyCPU, KeyDedicatedCPU, KeyMemory, KeyOvercommittedMemory, + KeyHugepagesPrefix + "2Mi", KeyHugepagesPrefix + "1Gi", +} + +func lookup(resolve func(string) (ResourceList, error), zeroKeys []string) functions.UnaryOp { + return func(arg ref.Val) ref.Val { + name, ok := arg.(types.String) + if !ok { + return types.MaybeNoSuchOverloadErr(arg) + } + rl, err := resolve(string(name)) + if err != nil { + return types.WrapErr(err) + } + out := make(map[string]any, len(rl)+len(zeroKeys)) + for _, k := range zeroKeys { + out[k] = apiservercel.Quantity{Quantity: resource.NewQuantity(0, resource.DecimalSI)} + } + for k, v := range rl { + q := v.DeepCopy() + out[k] = apiservercel.Quantity{Quantity: &q} + } + return types.DefaultTypeAdapter.NativeToValue(out) + } +} + +func mulQuantity(q, n ref.Val) ref.Val { + qq, ok := q.(apiservercel.Quantity) + if !ok { + return types.MaybeNoSuchOverloadErr(q) + } + nn, ok := n.(types.Int) + if !ok { + return types.MaybeNoSuchOverloadErr(n) + } + out := qq.Quantity.DeepCopy() + if !out.Mul(int64(nn)) { + return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn)) + } + return apiservercel.Quantity{Quantity: &out} +} + +func toQuantity(v ref.Val) (resource.Quantity, error) { + switch t := v.(type) { + case apiservercel.Quantity: + return t.Quantity.DeepCopy(), nil + case types.Int: + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Uint: + if uint64(t) > math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%d overflows a quantity", uint64(t)) + } + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Double: + f := float64(t) + if math.IsNaN(f) || math.IsInf(f, 0) { + return resource.Quantity{}, fmt.Errorf("not a finite number: %v", f) + } + // Rounding up would carry a negative figure smaller than a milli to + // zero, past the sign check. + if f < 0 { + return resource.Quantity{}, fmt.Errorf("negative reservation %v", f) + } + // float64(math.MaxInt64) rounds up to 2^63, so the bound is + // exclusive; past it the conversion would saturate into a smaller + // figure. + milli := math.Ceil(f * 1000) + if milli >= math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%v overflows a quantity", f) + } + return *resource.NewMilliQuantity(int64(milli), resource.DecimalSI), nil + case types.String: + q, err := resource.ParseQuantity(string(t)) + if err != nil { + return resource.Quantity{}, fmt.Errorf("parse %q as a quantity: %w", string(t), err) + } + return q, nil + default: + return resource.Quantity{}, fmt.Errorf("expression returned %s, expected an int, a double, a string or a quantity", v.Type().TypeName()) + } +} + +// checkTypes refuses a value whose JSON type the schema does not allow. It +// runs on normalized values: an integer is an int64, any other number a +// float64. A null is left to defaulting, which treats it as absent. +func checkTypes(v any, s *structuralschema.Structural, path string) error { + if s == nil || v == nil { + return nil + } + if s.XIntOrString { + switch v.(type) { + case int64, string: + return nil + } + return typeError(path, "an integer or a string", v) + } + // An untyped schema is defaulted as the shape its properties or items + // imply, so it is checked as that shape too. + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + } + } + switch effType { + case "object": + m, ok := v.(map[string]any) + if !ok { + return typeError(path, "an object", v) + } + for k, vv := range m { + ps, known := s.Properties[k] + switch { + case known: + if err := checkTypes(vv, &ps, path+"."+k); err != nil { + return err + } + case s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil: + if err := checkTypes(vv, s.AdditionalProperties.Structural, path+"."+k); err != nil { + return err + } + } + } + case "array": + sl, ok := v.([]any) + if !ok { + return typeError(path, "an array", v) + } + for i, vv := range sl { + if err := checkTypes(vv, s.Items, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + case "string": + if _, ok := v.(string); !ok { + return typeError(path, "a string", v) + } + case "integer": + if _, ok := v.(int64); !ok { + return typeError(path, "an integer", v) + } + case "number": + switch v.(type) { + case int64, float64: + default: + return typeError(path, "a number", v) + } + case "boolean": + if _, ok := v.(bool); !ok { + return typeError(path, "a boolean", v) + } + } + return nil +} + +func typeError(path, want string, got any) error { + if path == "" { + path = "." + } + return fmt.Errorf("values%s: expected %s, got %T", path, want, got) +} + +// normalizeNumbers returns a copy of v in which every JSON number is an int64 +// when it is integral and a float64 otherwise. Without it every number in a +// stored spec reaches CEL as a double, and replicas * quantity(...) would not +// type-check at runtime. +func normalizeNumbers(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, vv := range t { + out[k] = normalizeNumbers(vv) + } + return out + case []any: + out := make([]any, len(t)) + for i, vv := range t { + out[i] = normalizeNumbers(vv) + } + return out + case json.Number: + if i, err := t.Int64(); err == nil { + return i + } + // 2.0 and 2e0 are integers to the schema validator; spelled that + // way they fail Int64 and are classified as a float64 would be. + f, _ := t.Float64() + return normalizeNumbers(f) + case float64: + if t == math.Trunc(t) && math.Abs(t) < 1<<53 { + return int64(t) + } + return t + case int: + return int64(t) + default: + return v + } +} diff --git a/pkg/reservation/evaluator_test.go b/pkg/reservation/evaluator_test.go new file mode 100644 index 000000000..7aa7ef9a7 --- /dev/null +++ b/pkg/reservation/evaluator_test.go @@ -0,0 +1,525 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "os" + "sync" + "testing" + + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// The declarations below are test fixtures in the shape a kind would carry +// on its ApplicationDefinition. They are evaluated against the kinds' real +// shipped schemas so that defaulting is exercised with the defaults tenants +// actually get. + +var postgresDecl = &Declaration{ + Let: []Binding{{ + Name: "count", + // The autoscaler writes the Cluster's scale subresource, never the + // application, so the ceiling is what has to be reserved. + Expression: `values.autoscaling.enabled && values.autoscaling.maxReplicas > values.replicas + ? values.autoscaling.maxReplicas : values.replicas`, + }}, + Resources: map[string]string{ + // cozy-lib fills a preset per key, so each key is overridden on its own. + "cpu": `(has(values.resources.cpu) ? quantity(values.resources.cpu) : preset(values.resourcesPreset).cpu).mul(count)`, + "memory": `(has(values.resources.memory) ? quantity(values.resources.memory) : preset(values.resourcesPreset).memory).mul(count)`, + "storage": `quantity(values.size).mul(count)`, + }, +} + +var vmInstanceDecl = &Declaration{ + Let: []Binding{ + // vm-instance treats resources as all-or-nothing, and cpu is per socket. + {Name: "sized", Expression: `has(values.resources.cpu) && has(values.resources.sockets) && has(values.resources.memory)`}, + {Name: "it", Expression: `instanceType(values.instanceType)`}, + }, + Resources: map[string]string{ + "cpu": `sized ? int(values.resources.cpu) * int(values.resources.sockets) : it.cpu`, + "dedicated-cpu": `sized ? 0 : it['dedicated-cpu']`, + "memory": `sized ? quantity(values.resources.memory) : it.memory`, + "hugepages-2Mi": `sized ? 0 : it['hugepages-2Mi']`, + "hugepages-1Gi": `sized ? 0 : it['hugepages-1Gi']`, + "overcommitted-memory": `sized ? 0 : it['overcommitted-memory']`, + "services.loadbalancers": `values.external ? 1 : 0`, + }, +} + +var kubernetesNodesDecl = &Declaration{ + Let: []Binding{{Name: "it", Expression: `instanceType(values.instanceType)`}}, + Resources: map[string]string{ + "cpu": `it.cpu.mul(values.maxReplicas)`, + "memory": `it.memory.mul(values.maxReplicas)`, + }, +} + +func shippedSchema(t *testing.T, kind string) *structuralschema.Structural { + t.Helper() + raw, err := os.ReadFile("../../packages/system/" + kind + "-rd/cozyrds/" + kind + ".yaml") + if err != nil { + t.Fatal(err) + } + var def struct { + Spec struct { + Application struct { + OpenAPISchema string `json:"openAPISchema"` + } `json:"application"` + } `json:"spec"` + } + if err := yaml.Unmarshal(raw, &def); err != nil { + t.Fatal(err) + } + s, err := defaulting.BuildSchema(def.Spec.Application.OpenAPISchema) + if err != nil || s == nil { + t.Fatalf("%s schema: %v", kind, err) + } + return s +} + +func testEnvironment(t *testing.T) Environment { + t.Helper() + env, err := NewStaticEnvironment(shippedInstancetypes(t), map[string]ResourceList{ + "t1.nano": {"cpu": resource.MustParse("250m"), "memory": resource.MustParse("128Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + "t1.micro": {"cpu": resource.MustParse("500m"), "memory": resource.MustParse("256Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + }) + if err != nil { + t.Fatal(err) + } + return env +} + +func mustCompile(t *testing.T, decl *Declaration, kind string) *Evaluator { + t.Helper() + ev, err := Compile(decl, shippedSchema(t, kind)) + if err != nil { + t.Fatalf("compile %s: %v", kind, err) + } + return ev +} + +func rl(kv ...string) ResourceList { + out := ResourceList{} + for i := 0; i < len(kv); i += 2 { + out[kv[i]] = resource.MustParse(kv[i+1]) + } + return out +} + +func TestEvaluateKinds(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + cases := []struct { + name string + ev *Evaluator + values string + want ResourceList + }{ + { + // Two replicas of t1.micro with 10Gi each, all from schema defaults. + name: "postgres defaults", ev: postgres, values: `{}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + // The per-key fill: cpu from resources, memory from the preset. + name: "postgres partial resources on a preset", ev: postgres, + values: `{"replicas": 1, "resourcesPreset": "t1.nano", "resources": {"cpu": "1"}}`, + want: rl("cpu", "1", "memory", "128Mi", "storage", "10Gi"), + }, + { + name: "postgres integer cpu", ev: postgres, + values: `{"replicas": 1, "resources": {"cpu": 2, "memory": "1Gi"}}`, + want: rl("cpu", "2", "memory", "1Gi", "storage", "10Gi"), + }, + { + name: "postgres autoscaling reserves the ceiling", ev: postgres, + values: `{"autoscaling": {"enabled": true}}`, + want: rl("cpu", "3", "memory", "1536Mi", "storage", "60Gi"), + }, + { + name: "postgres autoscaling off reserves replicas", ev: postgres, + values: `{"autoscaling": {"enabled": false, "maxReplicas": 6}}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + name: "vm-instance default instance type", ev: vm, values: `{}`, + want: rl("cpu", "1", "memory", "4Gi"), + }, + { + name: "vm-instance dedicated type pins the emulator core", ev: vm, + values: `{"instanceType": "cx1.2xlarge"}`, + want: rl("dedicated-cpu", "9", "hugepages-2Mi", "16Gi"), + }, + { + name: "vm-instance overcommitted type", ev: vm, + values: `{"instanceType": "o1.large"}`, + want: rl("cpu", "2", "overcommitted-memory", "8Gi"), + }, + { + // cpu is cores per socket. + name: "vm-instance complete resources block", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "sockets": 2, "memory": "8Gi"}}`, + want: rl("cpu", "8", "memory", "8Gi"), + }, + { + // Without sockets the chart sizes the VM from its instance type. + name: "vm-instance cpu without sockets", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "memory": "8Gi"}}`, + want: rl("cpu", "1", "memory", "2Gi"), + }, + { + name: "vm-instance external", ev: vm, + values: `{"instanceType": "u1.small", "external": true}`, + want: rl("cpu", "1", "memory", "2Gi", "services.loadbalancers", "1"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.ev.EvaluateJSON([]byte(tc.values), env) + if err != nil { + t.Fatal(err) + } + assertList(t, tc.name, got, tc.want) + }) + } +} + +// A release a platform chart renders, such as ComputePlane's default md0 +// pool, carries only the keys the chart wrote. It has to reserve exactly what +// the same pool with its defaults written out reserves. +func TestEvaluateDefaultsStoredValues(t *testing.T) { + env := testEnvironment(t) + ev := mustCompile(t, kubernetesNodesDecl, "kubernetes-nodes") + + sparse, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0}`), env) + if err != nil { + t.Fatal(err) + } + explicit, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0, "maxReplicas": 10, "instanceType": "u1.medium"}`), env) + if err != nil { + t.Fatal(err) + } + assertList(t, "sparse vs explicit", sparse, explicit) + assertList(t, "sparse", sparse, rl("cpu", "10", "memory", "40Gi")) +} + +func TestEvaluateFailsClosed(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + if _, err := vm.EvaluateJSON([]byte(`{"instanceType": "u9.imaginary"}`), env); err == nil { + t.Error("unknown instance type: expected an error") + } else if !errors.Is(err, ErrNotFound) { + t.Errorf("unknown instance type: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"resourcesPreset": "t9.huge"}`), env); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown preset: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"replicas": "two"}`), env); err == nil { + t.Error("wrong value type: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{"size": "lots"}`), env); err == nil { + t.Error("unparseable quantity: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`[1, 2]`), env); err == nil { + t.Error("non-object values: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{}`), nil); err == nil { + t.Error("no environment: expected an error") + } + + unguarded, err := Compile(&Declaration{Resources: map[string]string{"cpu": `quantity(values.notInTheSchema)`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + if _, err := unguarded.EvaluateJSON([]byte(`{}`), env); err == nil { + t.Error("unguarded absent key: expected an error, not a zero") + } + + negative, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-1`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := negative.EvaluateJSON(nil, env); err == nil { + t.Error("negative reservation: expected an error") + } + + // A negative double too small to survive rounding to millis is still + // negative. + tiny, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-0.0001`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tiny.EvaluateJSON(nil, env); err == nil { + t.Error("small negative reservation: expected an error, not a zero") + } + + // A double past the range of an int64 in millis must not be clamped into + // a smaller figure. + for _, expr := range []string{`1e100`, `1e308`, `9.3e15`} { + huge, err := Compile(&Declaration{Resources: map[string]string{"cpu": expr}}, nil) + if err != nil { + t.Fatal(err) + } + if got, err := huge.EvaluateJSON(nil, env); err == nil { + t.Errorf("%s: expected an error, got %v", expr, got) + } + } + finite, err := Compile(&Declaration{Resources: map[string]string{"cpu": `1e3`, "memory": `0.25`}}, nil) + if err != nil { + t.Fatal(err) + } + got, err := finite.EvaluateJSON(nil, env) + if err != nil { + t.Fatal(err) + } + assertList(t, "finite doubles", got, rl("cpu", "1k", "memory", "250m")) +} + +func TestEvaluateRejectsValuesOfTheWrongType(t *testing.T) { + env := testEnvironment(t) + // Defaulting replaces a value whose type does not match the schema with + // the schema's own default, so the type is checked before it runs: + // otherwise this guard would read an erased resources object and + // reserve its fallback. + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{ + `{"resources": "invalid"}`, + `{"resources": [1]}`, + `{"resources": true}`, + `{"resources": {"cpu": true}}`, + `{"replicas": 2.5}`, + `{"replicas": "2"}`, + `{"external": "yes"}`, + `{"postgresql": {"sharedPreloadLibraries": "pg_prewarm"}}`, + `{"users": {"app": "admin"}}`, + } { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{}`: rl("cpu", "1"), + `{"resources": {}}`: rl("cpu", "1"), + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"resources": {"cpu": "500m"}}`: rl("cpu", "500m"), + `{"resources": null, "replicas": 3}`: rl("cpu", "1"), + `{"users": {"app": {"replication": true}}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestEvaluateAcceptsIntegralSpellings(t *testing.T) { + env := testEnvironment(t) + // The Kubernetes schema validator accepts an integral number however it + // is spelled, so the evaluator must too. + ev, err := Compile(&Declaration{Resources: map[string]string{"cpu": `values.replicas`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"replicas": 2}`, `{"replicas": 2.0}`, `{"replicas": 2e0}`, `{"replicas": 0.2e1}`} { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, rl("cpu", "2")) + } + for _, raw := range []string{`{"replicas": 2.5}`, `{"replicas": 25e-1}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } +} + +func TestEvaluateChecksUntypedSchemasByShape(t *testing.T) { + env := testEnvironment(t) + // BuildSchema does not require a structural schema to be typed, and + // defaulting treats an untyped node with properties as an object and one + // with items as an array, so the type check must as well. + s, err := defaulting.BuildSchema(`{"type":"object","properties":{ + "resources":{"properties":{"cpu":{"type":"integer"}}}, + "disks":{"items":{"type":"string"}}, + "labels":{"additionalProperties":{"type":"string"}}}}`) + if err != nil { + t.Fatal(err) + } + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources) && has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, s) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"resources": "invalid"}`, `{"resources": {"cpu": "2"}}`, `{"disks": "a"}`, `{"disks": [1]}`, `{"labels": "a"}`, `{"labels": {"a": 1}}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"disks": ["a"]}`: rl("cpu", "1"), + `{"labels": {"a": "b"}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestCompile(t *testing.T) { + if _, err := Compile(nil, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("nil declaration: got %v, want ErrNoDeclaration", err) + } + if _, err := Compile(&Declaration{}, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("empty declaration: got %v, want ErrNoDeclaration", err) + } + + exempt, err := Compile(&Declaration{Exempt: true}, nil) + if err != nil { + t.Fatal(err) + } + got, err := exempt.EvaluateJSON([]byte(`{"anything": 1}`), nil) + if err != nil || len(got) != 0 || !exempt.Exempt() { + t.Errorf("exempt: got %v, %v", got, err) + } + + invalid := map[string]*Declaration{ + "exempt with resources": {Exempt: true, Resources: map[string]string{"cpu": "1"}}, + "syntax error": {Resources: map[string]string{"cpu": "values.("}}, + "undeclared variable": {Resources: map[string]string{"cpu": "count"}}, + "empty key": {Resources: map[string]string{"": "1"}}, + "bad let name": {Let: []Binding{{Name: "a-b", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "let shadows values": {Let: []Binding{{Name: "values", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "duplicate let": {Let: []Binding{{Name: "a", Expression: "1"}, {Name: "a", Expression: "2"}}, Resources: map[string]string{"cpu": "a"}}, + } + for name, decl := range invalid { + if _, err := Compile(decl, nil); err == nil { + t.Errorf("%s: expected a compile error", name) + } + } +} + +func TestEvaluateIsConcurrencySafe(t *testing.T) { + env := testEnvironment(t) + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + catalog := shippedInstancetypes(t) + var wg sync.WaitGroup + for _, it := range []string{"u1.small", "cx1.2xlarge", "o1.large", "u1.medium"} { + want, err := ClassifyInstancetype(findInstancetype(t, catalog, it)) + if err != nil { + t.Fatal(err) + } + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + got, err := vm.EvaluateJSON([]byte(`{"instanceType": "`+it+`"}`), env) + if err != nil { + t.Error(err) + return + } + assertList(t, it, got, want) + }() + } + } + wg.Wait() +} + +func findInstancetype(t *testing.T, catalog []map[string]any, name string) map[string]any { + t.Helper() + for _, obj := range catalog { + if obj["metadata"].(map[string]any)["name"] == name { + return obj + } + } + t.Fatalf("%s is not shipped", name) + return nil +} + +func TestAggregate(t *testing.T) { + env := testEnvironment(t) + evaluators := map[string]*Evaluator{ + "VMInstance": mustCompile(t, vmInstanceDecl, "vm-instance"), + "Postgres": mustCompile(t, postgresDecl, "postgres"), + } + lookup := func(kind, group string) (*Evaluator, error) { + if group != "apps.cozystack.io" { + return nil, errors.New("unknown group") + } + if ev, ok := evaluators[kind]; ok { + return ev, nil + } + return nil, ErrNoDeclaration + } + apps := []Application{ + {Namespace: "tenant-a", Name: "vm1", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-a", Name: "vm2", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small", "external": true}`)}, + {Namespace: "tenant-a", Name: "db", Kind: "Postgres", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + {Namespace: "tenant-b", Name: "vm3", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-b", Name: "broken", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "gone"}`)}, + {Namespace: "tenant-b", Name: "undeclared", Kind: "Redis", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + } + sums, failures := Aggregate(apps, lookup, env) + assertList(t, "tenant-a", sums["tenant-a"], rl("cpu", "3", "memory", "4608Mi", "storage", "20Gi", "services.loadbalancers", "1")) + assertList(t, "tenant-b", sums["tenant-b"], rl("cpu", "1", "memory", "2Gi")) + if len(failures) != 2 { + t.Fatalf("expected 2 failures, got %v", failures) + } + if failures[0].Application.Name != "broken" || !errors.Is(failures[0].Err, ErrNotFound) { + t.Errorf("first failure: %v", failures[0]) + } + if failures[1].Application.Name != "undeclared" || !errors.Is(failures[1].Err, ErrNoDeclaration) { + t.Errorf("second failure: %v", failures[1]) + } +} + +func TestResourceListArithmetic(t *testing.T) { + a := rl("cpu", "2", "memory", "4Gi") + b := a.DeepCopy() + b.Add(rl("cpu", "1", "storage", "10Gi")) + assertList(t, "copy unaffected", a, rl("cpu", "2", "memory", "4Gi")) + assertList(t, "add", b, rl("cpu", "3", "memory", "4Gi", "storage", "10Gi")) + b.Sub(rl("cpu", "4", "services.loadbalancers", "1")) + assertList(t, "sub", b, rl("cpu", "-1", "memory", "4Gi", "storage", "10Gi", "services.loadbalancers", "-1")) + if keys := b.Keys(); len(keys) != 4 || keys[0] != "cpu" || keys[3] != "storage" { + t.Errorf("keys not sorted: %v", keys) + } +} diff --git a/pkg/reservation/instancetype.go b/pkg/reservation/instancetype.go new file mode 100644 index 000000000..b4373164f --- /dev/null +++ b/pkg/reservation/instancetype.go @@ -0,0 +1,148 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// EmulatorThreadCompleteToEvenParityAnnotation, set in an instance type's +// spec.annotations, makes KubeVirt give the isolated emulator thread one or +// two cores, whichever makes the pinned total even. KubeVirt copies those +// annotations onto the VMI and tests the key for presence, so its value is +// irrelevant. +const EmulatorThreadCompleteToEvenParityAnnotation = "alpha.kubevirt.io/EmulatorThreadCompleteToEvenParity" + +// ClassifyInstancetype resolves a VirtualMachineClusterInstancetype into the +// class-keyed reservation of one VM of that type. +// +// The class is read from the type's spec, never from its name, so a type an +// operator adds classifies itself. Cores include the emulator and +// supplemental I/O threads a dedicated type pins, because those are fixed +// properties of the type a tenant can see before ordering; the launcher's +// memory overhead is computed by virt-controller at runtime and is platform +// margin, not part of the reservation. +func ClassifyInstancetype(obj map[string]any) (ResourceList, error) { + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + + guestCPU, err := nestedInt(obj, "spec", "cpu", "guest") + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest: %w", name, err) + } + if guestCPU <= 0 { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest must be positive, got %d", name, guestCPU) + } + guestMemoryRaw, found, err := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "guest") + if err != nil || !found { + return nil, fmt.Errorf("instance type %q: spec.memory.guest is required", name) + } + guestMemory, err := parseQuantity(guestMemoryRaw) + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.memory.guest: %w", name, err) + } + + out := ResourceList{} + + dedicated, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "dedicatedCPUPlacement") + if dedicated { + cores := guestCPU + if isolate, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "isolateEmulatorThread"); isolate { + cores++ + annotations, _, _ := unstructured.NestedMap(obj, "spec", "annotations") + if _, parity := annotations[EmulatorThreadCompleteToEvenParityAnnotation]; parity && cores%2 == 1 { + cores++ + } + } + if policy, _, _ := unstructured.NestedString(obj, "spec", "ioThreadsPolicy"); policy == "supplementalPool" { + extra, err := nestedInt(obj, "spec", "ioThreads", "supplementalPoolThreadCount") + if err != nil { + return nil, fmt.Errorf("instance type %q: ioThreadsPolicy supplementalPool: spec.ioThreads.supplementalPoolThreadCount: %w", name, err) + } + cores += extra + } + out[KeyDedicatedCPU] = *resource.NewQuantity(cores, resource.DecimalSI) + } else { + out[KeyCPU] = *resource.NewQuantity(guestCPU, resource.DecimalSI) + } + + pageSize, hasHugepages, _ := unstructured.NestedString(obj, "spec", "memory", "hugepages", "pageSize") + overcommit, hasOvercommit, _ := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "overcommitPercent") + overcommitted := hasOvercommit && overcommit != nil && fmt.Sprint(overcommit) != "0" + switch { + case hasHugepages && overcommitted: + return nil, fmt.Errorf("instance type %q: hugepages and overcommitPercent are mutually exclusive", name) + case hasHugepages: + if pageSize == "" { + return nil, fmt.Errorf("instance type %q: spec.memory.hugepages.pageSize is empty", name) + } + out[KeyHugepagesPrefix+pageSize] = guestMemory + case overcommitted: + // The launcher requests less than the guest here, but the tenant + // declared the guest figure, and the same page cannot be granted as + // both overcommitted and ordinary memory. + out[KeyOvercommittedMemory] = guestMemory + default: + out[KeyMemory] = guestMemory + } + return out, nil +} + +func nestedInt(obj map[string]any, fields ...string) (int64, error) { + v, found, err := unstructured.NestedFieldNoCopy(obj, fields...) + if err != nil { + return 0, err + } + if !found || v == nil { + return 0, fmt.Errorf("is required") + } + switch n := v.(type) { + case int64: + return n, nil + case int: + return int64(n), nil + case int32: + return int64(n), nil + case float64: + if n != float64(int64(n)) { + return 0, fmt.Errorf("must be an integer, got %v", n) + } + return int64(n), nil + default: + return 0, fmt.Errorf("must be an integer, got %T", v) + } +} + +func parseQuantity(v any) (resource.Quantity, error) { + switch q := v.(type) { + case string: + return resource.ParseQuantity(q) + case int64: + return *resource.NewQuantity(q, resource.BinarySI), nil + case int: + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + case float64: + if q != float64(int64(q)) { + return resource.Quantity{}, fmt.Errorf("fractional byte count %v", q) + } + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + default: + return resource.Quantity{}, fmt.Errorf("must be a quantity, got %T", v) + } +} diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go new file mode 100644 index 000000000..0d072b548 --- /dev/null +++ b/pkg/reservation/instancetype_test.go @@ -0,0 +1,259 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" +) + +// shippedInstancetypes decodes every VirtualMachineClusterInstancetype the +// platform ships, so that the classification is pinned against the real +// catalog rather than against hand-written fixtures. +func shippedInstancetypes(t *testing.T) []map[string]any { + t.Helper() + files, err := filepath.Glob("../../packages/system/kubevirt-instancetypes/templates/*.yaml") + if err != nil || len(files) == 0 { + t.Fatalf("no instance type templates found: %v", err) + } + var out []map[string]any + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + t.Fatal(err) + } + dec := utilyaml.NewYAMLOrJSONDecoder(fh, 4096) + for { + var obj map[string]any + if err := dec.Decode(&obj); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("%s: %v", f, err) + } + if obj["kind"] == "VirtualMachineClusterInstancetype" { + out = append(out, obj) + } + } + fh.Close() + } + return out +} + +func TestClassifyShippedInstancetypes(t *testing.T) { + // The expected class per series is written from what each series is, + // not derived from the spec, so a name-based rule and a spec-based rule + // disagreeing shows up here. m1 is shared CPU with hugepages, n1 a + // dedicated series with hugepages, o1 overcommits memory. + type class struct{ cpu, memory string } + want := map[string]class{ + "u1": {KeyCPU, KeyMemory}, + "gn1": {KeyCPU, KeyMemory}, + "m1": {KeyCPU, "hugepages"}, + "o1": {KeyCPU, KeyOvercommittedMemory}, + "d1": {KeyDedicatedCPU, KeyMemory}, + "cx1": {KeyDedicatedCPU, "hugepages"}, + "n1": {KeyDedicatedCPU, "hugepages"}, + "rt1": {KeyDedicatedCPU, "hugepages"}, + } + + types := shippedInstancetypes(t) + if len(types) < 60 { + t.Fatalf("expected the full shipped catalog, decoded only %d instance types", len(types)) + } + seen := map[string]bool{} + for _, obj := range types { + name := obj["metadata"].(map[string]any)["name"].(string) + series := strings.SplitN(name, ".", 2)[0] + seen[series] = true + w, ok := want[series] + if !ok { + t.Errorf("%s: series %q has no expected class; add it to this table", name, series) + continue + } + rl, err := ClassifyInstancetype(obj) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + if len(rl) != 2 { + t.Errorf("%s: expected one CPU class and one memory class, got %v", name, rl) + } + if _, ok := rl[w.cpu]; !ok { + t.Errorf("%s: expected CPU class %q, got %v", name, w.cpu, rl) + } + if w.memory == "hugepages" { + found := false + for k := range rl { + found = found || strings.HasPrefix(k, KeyHugepagesPrefix) + } + if !found { + t.Errorf("%s: expected a hugepages class, got %v", name, rl) + } + } else if _, ok := rl[w.memory]; !ok { + t.Errorf("%s: expected memory class %q, got %v", name, w.memory, rl) + } + } + for series := range want { + if !seen[series] { + t.Errorf("series %q is no longer shipped; drop it from this table", series) + } + } +} + +func TestClassifyShippedFigures(t *testing.T) { + byName := map[string]map[string]any{} + for _, obj := range shippedInstancetypes(t) { + byName[obj["metadata"].(map[string]any)["name"].(string)] = obj + } + cases := []struct { + name string + want ResourceList + }{ + // The KubeVirt u1.small is 2Gi; the cozy-lib preset of the same name + // is 4Gi. The two catalogs share names, not figures. + {"u1.small", ResourceList{KeyCPU: resource.MustParse("1"), KeyMemory: resource.MustParse("2Gi")}}, + // Eight guest vCPUs on nine pinned cores: the isolated emulator + // thread takes a core nobody else can be given. + {"cx1.2xlarge", ResourceList{KeyDedicatedCPU: resource.MustParse("9"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + {"d1.large", ResourceList{KeyDedicatedCPU: resource.MustParse("3"), KeyMemory: resource.MustParse("8Gi")}}, + {"m1.large", ResourceList{KeyCPU: resource.MustParse("2"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + } + for _, tc := range cases { + obj, ok := byName[tc.name] + if !ok { + t.Fatalf("%s is not shipped", tc.name) + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + assertList(t, tc.name, got, tc.want) + } +} + +func TestClassifyEmulatorThreadParity(t *testing.T) { + // KubeVirt copies an instance type's spec.annotations onto the VMI and + // pins the extra parity core when the VMI carries the annotation, so the + // figure is a property of each type, not of the catalog. + dedicated := func(guest int64, annotations map[string]any) map[string]any { + spec := map[string]any{ + "cpu": map[string]any{"guest": guest, "dedicatedCPUPlacement": true, "isolateEmulatorThread": true}, + "memory": map[string]any{"guest": "1Gi"}, + } + if annotations != nil { + spec["annotations"] = annotations + } + return map[string]any{"metadata": map[string]any{"name": "test"}, "spec": spec} + } + parity := map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: ""} + cases := []struct { + guest int64 + annotations map[string]any + want int64 + }{ + {guest: 8, want: 9}, + {guest: 8, annotations: parity, want: 10}, + {guest: 8, annotations: map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: "false"}, want: 10}, + {guest: 8, annotations: map[string]any{"example.com/unrelated": ""}, want: 9}, + {guest: 3, want: 4}, + {guest: 3, annotations: parity, want: 4}, + } + for _, tc := range cases { + got, err := ClassifyInstancetype(dedicated(tc.guest, tc.annotations)) + if err != nil { + t.Fatal(err) + } + cores := got[KeyDedicatedCPU] + if cores.Value() != tc.want { + t.Errorf("guest %d annotations %v: got %s dedicated cores, want %d", tc.guest, tc.annotations, cores.String(), tc.want) + } + } + + // Only the type's own annotations reach the VMI; metadata.annotations + // describe the object and are not copied. + onMetadata := dedicated(8, nil) + onMetadata["metadata"].(map[string]any)["annotations"] = parity + got, err := ClassifyInstancetype(onMetadata) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 9 { + t.Errorf("annotation on metadata: got %s dedicated cores, want 9", cores.String()) + } +} + +func TestClassifySupplementalIOThreads(t *testing.T) { + obj := map[string]any{ + "metadata": map[string]any{"name": "test"}, + "spec": map[string]any{ + "cpu": map[string]any{"guest": int64(4), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "ioThreads": map[string]any{"supplementalPoolThreadCount": int64(2)}, + "memory": map[string]any{"guest": "1Gi"}, + }, + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 6 { + t.Errorf("got %s dedicated cores, want 6", cores.String()) + } +} + +func TestClassifyRejectsIncompleteTypes(t *testing.T) { + cases := map[string]map[string]any{ + "no guest cpu": {"spec": map[string]any{"memory": map[string]any{"guest": "1Gi"}}}, + "no guest memory": {"spec": map[string]any{"cpu": map[string]any{"guest": int64(1)}}}, + "hugepages and overcommit": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1)}, + "memory": map[string]any{"guest": "1Gi", "overcommitPercent": int64(50), "hugepages": map[string]any{"pageSize": "2Mi"}}, + }}, + "supplementalPool without a count": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "memory": map[string]any{"guest": "1Gi"}, + }}, + } + for name, obj := range cases { + if got, err := ClassifyInstancetype(obj); err == nil { + t.Errorf("%s: expected an error, got %v", name, got) + } + } +} + +func assertList(t *testing.T, label string, got, want ResourceList) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s: got %v, want %v", label, got, want) + return + } + for k, w := range want { + g, ok := got[k] + if !ok || g.Cmp(w) != 0 { + t.Errorf("%s: %s = %s, want %s (full: %v)", label, k, g.String(), w.String(), got) + } + } +} diff --git a/pkg/reservation/types.go b/pkg/reservation/types.go new file mode 100644 index 000000000..1a01a0763 --- /dev/null +++ b/pkg/reservation/types.go @@ -0,0 +1,113 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package reservation computes what an application consumes from a tenant's +// quota budget, from its declared values, before anything runs. +// +// A reservation is denominated in the shorthand keys a Tenant's +// resourceQuotas uses (cpu, memory, storage, services.loadbalancers, ...), +// never in rendered pod units: no allocation ratio is applied and no +// virtualization overhead is added. The evaluator holds no per-kind +// knowledge; each kind describes its consumption in a Declaration carried by +// its ApplicationDefinition, so a kind shipped from an out-of-tree catalog +// participates without a change here. +package reservation + +import ( + "sort" + + "k8s.io/apimachinery/pkg/api/resource" +) + +// Quota keys for KubeVirt instance-type classes. A shared vCPU and a pinned +// core are not the same capacity, nor are ordinary, hugepage-backed and +// overcommitted memory, so each class counts against its own key. +const ( + KeyCPU = "cpu" + KeyDedicatedCPU = "dedicated-cpu" + KeyMemory = "memory" + KeyOvercommittedMemory = "overcommitted-memory" + // KeyHugepagesPrefix is followed by the page size, e.g. "hugepages-2Mi", + // matching the corev1 resource name for the same pages. + KeyHugepagesPrefix = "hugepages-" +) + +// Declaration is how a kind's reservation is computed from its values. +// +// Exactly one of Exempt or Resources is meaningful: a kind that consumes +// nothing says so with Exempt rather than with an empty Resources map, so +// that a zero reservation is always a reviewed statement and never an +// omission. +type Declaration struct { + // Exempt declares that the kind consumes nothing from a tenant's budget. + Exempt bool `json:"exempt,omitempty"` + + // Let binds intermediate CEL expressions, evaluated in order, each one + // visible to the later bindings and to every Resources expression. + Let []Binding `json:"let,omitempty"` + + // Resources maps a quota key to the CEL expression computing the + // amount of it the application reserves. An expression may return an + // int, a double, a quantity string or a quantity. + Resources map[string]string `json:"resources,omitempty"` +} + +// Binding names one intermediate CEL expression of a Declaration. +type Binding struct { + Name string `json:"name"` + Expression string `json:"expression"` +} + +// ResourceList is a reservation, keyed by quota key. +type ResourceList map[string]resource.Quantity + +// Add adds other into l in place. +func (l ResourceList) Add(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Add(v) + l[k] = cur + } +} + +// Sub subtracts other from l in place. A key absent from l goes negative, +// which is what the delta of an Update needs. +func (l ResourceList) Sub(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Sub(v) + l[k] = cur + } +} + +// DeepCopy returns an independent copy of l. +func (l ResourceList) DeepCopy() ResourceList { + out := make(ResourceList, len(l)) + for k, v := range l { + out[k] = v.DeepCopy() + } + return out +} + +// Keys returns the keys of l in a stable order, for deterministic messages. +func (l ResourceList) Keys() []string { + keys := make([]string, 0, len(l)) + for k := range l { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}