From a2d0e4cb5058092a66cb3173503850c64ba9fab0 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 06:07:07 +0000 Subject: [PATCH 01/20] fix(cilium): pass the CiliumNode list to jq on stdin in the IPAM hook The hook fed the whole `kubectl get ciliumnodes -o json` to jq as one argument. Cilium copies every Node label onto its CiliumNode, 22 to 30 KiB per node on Talos or with NFD and GPU labels, so the list passes the 128 KiB Linux allows a single argument within six nodes. jq then fails to start with "Argument list too long", the hook Job fails before it moves any node, and the cilium HelmRelease loops on UpgradeFailed with the HelmReleases behind it stuck in DependencyNotReady. The hook renders on every upgrade of cilium in cluster-pool mode, so a cluster that already finished the migration hits the same failure on its next values change. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- hack/cilium-ipam-migration.bats | 27 +++++++++++++++++++ .../system/cilium/files/ipam-migration.sh | 10 ++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hack/cilium-ipam-migration.bats b/hack/cilium-ipam-migration.bats index c54db50ab..53053747a 100644 --- a/hack/cilium-ipam-migration.bats +++ b/hack/cilium-ipam-migration.bats @@ -488,6 +488,33 @@ pod_cidrs_of() { rm -rf "$STATE" } +@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" { + init_state + add_ip web-0.tenant-c ovn-default 10.244.1.135 + add_ip web-1.tenant-c ovn-default 10.244.4.135 + add_ip db-0.tenant-c ovn-default 10.244.9.9 + # Cilium copies every Node label onto the CiliumNode, which takes six real + # ones past the 128 KiB Linux allows a single argument. One annotation stands + # in for the labels and keeps the list over that limit even printed compact. + for i in 1 2 3 4 5 6; do + add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135" + jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp" + mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json" + done + [ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ] + "$SCRIPT" > "$STATE/out" 2>&1 + # Each pod is named with the node whose Ingress IP it holds, and a pod on an + # address that is no node's Ingress IP is not named at all. + grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out" + grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out" + if grep -q "db-0.tenant-c" "$STATE/out"; then + cat "$STATE/out" + false + fi + [ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ] + rm -rf "$STATE" +} + @test "a cluster without kube-ovn skips the exclusion and still moves its nodes" { init_state rm "$STATE/subnet.json" diff --git a/packages/system/cilium/files/ipam-migration.sh b/packages/system/cilium/files/ipam-migration.sh index 002a1bf78..9345c19f6 100755 --- a/packages/system/cilium/files/ipam-migration.sh +++ b/packages/system/cilium/files/ipam-migration.sh @@ -240,10 +240,12 @@ exclude_from_kube_ovn() { fi # A pod that holds a node's current Ingress IP keeps the address after the # move, and that node keeps it in its ipcache as reserved:ingress, so the pod - # stays unreachable from there until it is recreated. - jq -r --argjson cn "$ciliumnodes" ' - ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress - | .[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ipobjects" \ + # stays unreachable from there until it is recreated. The CiliumNode list goes + # in on stdin, because the Node labels Cilium copies onto every CiliumNode take + # it past the 128 KiB a single argument may hold within a handful of nodes. + jq -rn 'input as $cn + | ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress + | input[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ciliumnodes"$'\n'"$ipobjects" \ | while read -r ipname ip node; do if ! in_any "$POOL" "$ip"; then log "$ipname holds $ip, the Ingress IP of $node before the move. Recreate it once the run is done, $node keeps that address as reserved:ingress" From 7b7444feaf5508eaf62767b8c999468558732d1b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 06:16:36 +0000 Subject: [PATCH 02/20] fix(kubernetes): move the OIDC users out of the bootstrap Job's script The bootstrap Job wrote a ClusterRoleBinding per oidc.users entry into its script, and the script reaches the container as one `sh -c` argument. Linux caps a single argument at 128 KiB, which about 240 users cross at roughly 530 bytes each. The container then cannot start, the post-install and post-upgrade hook fails, and the tenant cluster's HelmRelease stays unready. The values schema puts no bound on oidc.users, so a shared cluster reaches that size with ordinary per-person entries. The bindings and the list of their names now reach the Job through a ConfigMap it mounts, so the script is the same size for any list. Names, labels and roles of the bindings are unchanged, so the bindings already in a tenant cluster are updated in place and none is orphaned. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- hack/kubernetes-oidc-bindings.bats | 187 +++++++++++++++++ .../templates/oidc-rbac-bindings.yaml | 42 ++++ .../kubernetes/templates/oidc-rbac-job.yaml | 57 +++--- packages/apps/kubernetes/tests/oidc_test.yaml | 193 +++++++++++++++++- 4 files changed, 439 insertions(+), 40 deletions(-) create mode 100644 hack/kubernetes-oidc-bindings.bats create mode 100644 packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml diff --git a/hack/kubernetes-oidc-bindings.bats b/hack/kubernetes-oidc-bindings.bats new file mode 100644 index 000000000..f37273802 --- /dev/null +++ b/hack/kubernetes-oidc-bindings.bats @@ -0,0 +1,187 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[] +# list inside its container command. +# +# The Job's script reaches the container as one `sh -c` argument, and Linux caps +# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per +# user written into that script, about 240 users take it past the cap: the +# container cannot start, the post-install/post-upgrade hook fails, and the +# tenant cluster's HelmRelease stays unready. The bindings reach the Job through +# a mounted ConfigMap instead, so the script is the same size for any list. +# +# The last three tests execute the rendered script against a fake kubectl, since +# no assertion on the script's text says what the prune step does with the list: +# a condition forced either way passes every helm-unittest case. +# +# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats. +# ----------------------------------------------------------------------------- + +load test_helper + +USERS=300 + +# render_users renders the chart in System mode with +# generated users and leaves one JSON document per object in /rendered.json. +render_users() { + jq -n --argjson n "$2" ' + {oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \ + > "$1/users.yaml" + helm template bindings packages/apps/kubernetes \ + --namespace tenant-test \ + --values packages/apps/kubernetes/tests/values/oidc-system.yaml \ + --values "$1/users.yaml" \ + > "$1/rendered.yaml" + yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json" +} + +# render_job renders the chart in CustomConfig mode and unpacks +# what the bootstrap Job runs and mounts into : script.sh is its `sh -c` +# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an +# empty value an empty file, as the kubelet writes it. +render_job() { + jq -n --argjson users "$2" ' + {_cluster: {"oidc-enabled": "false"}, + oidc: {mode: "CustomConfig", + customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"}, + users: $users}}' > "$1/values.yaml" + helm template bindings packages/apps/kubernetes \ + --namespace tenant-test \ + --values packages/apps/kubernetes/tests/values/oidc-system.yaml \ + --values "$1/values.yaml" \ + > "$1/rendered.yaml" + yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json" + jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap") + | .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh" + mkdir -p "$1/bindings" + jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml" + jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt" +} + +# fake_kubectl puts a kubectl first on PATH that logs every call and +# answers the script's list of existing bindings with . +fake_kubectl() { + mkdir -p "$1/bin" + printf '%s' "$2" > "$1/existing" + cat > "$1/bin/kubectl" <<'KEOF' +#!/bin/sh +echo "$*" >> "$KLOG" +case "$*" in + *"get clusterrolebindings"*) cat "$EXISTING" ;; +esac +KEOF + chmod +x "$1/bin/kubectl" +} + +# run_job runs the unpacked script as the container does and leaves its +# status in $rc and its output in /out. Run by bash, which has the pipefail +# the script sets and which is what the unit lane runs under; the container's own +# shell is not exercised. +run_job() { + : > "$1/kubectl.log" + echo kubeconfig > "$1/kubeconfig" + rc=0 + env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \ + KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \ + OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \ + bash "$1/script.sh" > "$1/out" 2>&1 || rc=$? +} + +@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" { + dir=$(mktemp -d) + render_users "$dir" "$USERS" + jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap") + | .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script" + size=$(wc -c < "$dir/script") + [ "$size" -gt 0 ] + [ "$size" -lt 131072 ] + if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then + echo "the Job script still carries users[] entries" >&2 + false + fi + rm -rf "$dir" +} + +@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" { + dir=$(mktemp -d) + render_users "$dir" "$USERS" + jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml" + jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt" + [ -s "$dir/crbs.yaml" ] + yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json" + + [ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ] + [ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ] + [ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ] + + # The name is what the prune step and every binding already in a tenant cluster + # go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of + # sha256("-"). + for i in 0 1 6 7; do + email="user$i@example.org" + name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)" + [ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ] + grep -q -x -F "$name" "$dir/desired.txt" + done + + # The list the prune step compares against is exactly the applied names. + [ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ] + rm -rf "$dir" +} + +@test "the Job applies the mounted bindings and deletes only the bindings no user has" { + dir=$(mktemp -d) + render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]' + alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)" + bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)" + fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef" + run_job "$dir" + if [ "$rc" -ne 0 ]; then + cat "$dir/out" + false + fi + grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log" + grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log" + grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log" + if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then + echo "the Job deleted a binding a user still has" >&2 + false + fi + [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ] + rm -rf "$dir" +} + +@test "a release with no users applies nothing and deletes every binding it left behind" { + dir=$(mktemp -d) + render_job "$dir" '[]' + fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef" + run_job "$dir" + if [ "$rc" -ne 0 ]; then + cat "$dir/out" + false + fi + if grep -q -F -- ' apply ' "$dir/kubectl.log"; then + echo "the Job ran kubectl apply with no bindings to apply" >&2 + false + fi + [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ] + rm -rf "$dir" +} + +@test "the Job stops before deleting anything when the desired list is not mounted" { + dir=$(mktemp -d) + render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]' + fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef" + rm "$dir/bindings/desired.txt" + run_job "$dir" + [ "$rc" -ne 0 ] + if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then + cat "$dir/kubectl.log" + false + fi + rm -rf "$dir" +} diff --git a/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml new file mode 100644 index 000000000..a8517da4f --- /dev/null +++ b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml @@ -0,0 +1,42 @@ +{{- /* + The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies + inside the tenant cluster, one per `oidc.users` entry, and the list of their + names it prunes against. They travel in a ConfigMap the Job mounts and not as + text inside its script: the script reaches the container as a single `sh -c` + argument, Linux caps one argument at 128 KiB, and a binding per user crosses + that cap at about 240 users, after which the container cannot start. + + A release-owned object, not a hook resource, so it exists before the + post-install and post-upgrade hooks run. It is rendered only where users are + bound; at `mode: None` the Job takes its early branch and the volume is + optional. +*/}} +{{- if ne .Values.oidc.mode "None" }} +{{- $desired := "" }} +{{- $bindings := "" }} +{{- range .Values.oidc.users }} +{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }} +{{- $role := "view" }} +{{- if eq .role "admin" }} +{{- $role = "cluster-admin" }} +{{- end }} +{{- $desired = printf "%s%s\n" $desired $name }} +{{- $binding := dict + "apiVersion" "rbac.authorization.k8s.io/v1" + "kind" "ClusterRoleBinding" + "metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name)) + "roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role) + "subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }} +{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-oidc-bindings + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +data: + {{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml index 8672ce124..8556d092b 100644 --- a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml +++ b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml @@ -25,7 +25,9 @@ - `mode: System | CustomConfig`: - Apply one ClusterRoleBinding per `users[]` entry inside the tenant cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the - literal email matched against the `email` claim). + literal email matched against the `email` claim). The bindings are + rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts, + not into the script below. - Prune stale CRBs labelled by this release. - `mode: System`: patch the pre-rendered `-oidc-kubeconfig` Secret with a real kubeconfig wired to `kubectl oidc-login` @@ -168,6 +170,8 @@ spec: value: {{ $issuer | quote }} - name: OIDC_SERVER_URL value: {{ printf "https://%s:443" $externalHost | quote }} + - name: OIDC_BINDINGS_DIR + value: /etc/oidc-bindings resources: requests: cpu: 10m @@ -254,46 +258,32 @@ spec: sleep 5 done + # A missing list must stop the Job here: grep on a file that is not + # there exits 2, `! grep` is then true, and the prune below would + # delete every binding of the release. + BINDINGS=$OIDC_BINDINGS_DIR + if [ ! -f "$BINDINGS/desired.txt" ]; then + echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2 + exit 1 + fi + echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..." - DESIRED_FILE=$(mktemp) - {{- range .Values.oidc.users }} - {{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }} - {{- $crbName := printf "cozystack-oidc-%s" $hash }} - {{- $role := "view" }} - {{- if eq .role "admin" }} - {{- $role = "cluster-admin" }} - {{- end }} - echo "{{ $crbName }}" >> "$DESIRED_FILE" - cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: {{ $crbName }} - labels: - app.kubernetes.io/managed-by: cozystack-oidc - app.kubernetes.io/instance: {{ $.Release.Name }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ $role }} - subjects: - - kind: User - apiGroup: rbac.authorization.k8s.io - name: {{ .email | quote }} - EOF - {{- end }} + # kubectl apply fails on a file with no objects, and a release with + # no users has none. + if [ -s "$BINDINGS/crbs.yaml" ]; then + kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml" + fi echo "Pruning stale ClusterRoleBindings labelled by this release..." EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \ --selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \ --output=jsonpath='{.items[*].metadata.name}') for crb in $EXISTING; do - if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then + if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then echo " removing $crb" kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found fi done - rm -f "$DESIRED_FILE" if [ "$OIDC_MODE" = "System" ]; then echo "Patching the OIDC kubeconfig Secret on the management cluster..." @@ -349,6 +339,9 @@ spec: - name: kubeconfig mountPath: /etc/kubernetes/kubeconfig readOnly: true + - name: oidc-bindings + mountPath: /etc/oidc-bindings + readOnly: true - name: tmp mountPath: /tmp volumes: @@ -356,6 +349,10 @@ spec: secret: secretName: {{ .Release.Name }}-admin-kubeconfig optional: true + - name: oidc-bindings + configMap: + name: {{ .Release.Name }}-oidc-bindings + optional: true - name: tmp emptyDir: {} {{- /* diff --git a/packages/apps/kubernetes/tests/oidc_test.yaml b/packages/apps/kubernetes/tests/oidc_test.yaml index 0623063e7..8511e392c 100644 --- a/packages/apps/kubernetes/tests/oidc_test.yaml +++ b/packages/apps/kubernetes/tests/oidc_test.yaml @@ -2561,7 +2561,73 @@ tests: value: "true" documentIndex: 3 - - it: bootstrap Job command builds a CRB for each users[] entry + - it: the bindings ConfigMap holds a CRB for each users[] entry + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: + - email: alice@example.org + role: admin + - email: bob@example.org + role: view + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: kubernetes-app-oidc-bindings + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: alice@example.org' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: bob@example.org' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'app.kubernetes.io/managed-by: cozystack-oidc' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'app.kubernetes.io/instance: kubernetes-app' + + - it: the binding names and roles stay what the Job script used to render + # The prune step and every binding already in a tenant cluster go by these + # names: cozystack-oidc- plus the first 16 hex digits of + # sha256("-"). A different name leaves the old binding + # behind next to the new one. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: + - email: alice@example.org + role: admin + - email: bob@example.org + role: view + asserts: + - equal: + path: data["desired.txt"] + value: "cozystack-oidc-0001a28136ca1a02\ncozystack-oidc-ec8dfb1cbc75e699\n" + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: cozystack-oidc-0001a28136ca1a02\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: alice@example.org\n' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: cozystack-oidc-ec8dfb1cbc75e699\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: view\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: bob@example.org\n' + + - it: the bootstrap Job mounts the bindings ConfigMap and carries no users[] entry in its script + # The script is one `sh -c` argument and Linux caps one argument at 128 KiB: + # a binding per user in it stops the container from starting at about 240 + # users. hack/kubernetes-oidc-bindings.bats renders 300 users to hold the + # size itself. release: name: kubernetes-app namespace: tenant-foo @@ -2575,23 +2641,130 @@ tests: role: admin - email: bob@example.org role: view + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: oidc-bindings + configMap: + name: kubernetes-app-oidc-bindings + optional: true + documentIndex: 3 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: oidc-bindings + mountPath: /etc/oidc-bindings + readOnly: true + documentIndex: 3 + # The script reads the directory from the environment, which is also how + # hack/kubernetes-oidc-bindings.bats points it at a scratch directory. + - contains: + path: spec.template.spec.containers[0].env + content: + name: OIDC_BINDINGS_DIR + value: /etc/oidc-bindings + documentIndex: 3 + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'alice@example\.org' + documentIndex: 3 + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'bob@example\.org' + documentIndex: 3 + + - it: the bootstrap Job stops before pruning when the desired list is not mounted + # `! grep -F` on a file that is not there is true, so a missing mount would + # delete every binding of the release. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-job.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System asserts: - matchRegex: path: spec.template.spec.containers[0].command[2] - pattern: 'name: "alice@example.org"' + pattern: '(?s)if \[ ! -f "\$BINDINGS/desired\.txt" \]; then.*?exit 1\n\s*fi.*Pruning stale ClusterRoleBindings' documentIndex: 3 + + - it: the bootstrap Job applies the bindings file only when it has objects in it + # kubectl apply exits non-zero on a file with no objects, and a release + # with no users has none. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-job.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + asserts: - matchRegex: path: spec.template.spec.containers[0].command[2] - pattern: 'name: "bob@example.org"' + pattern: 'if \[ -s "\$BINDINGS/crbs\.yaml" \]; then\n\s+kubectl --kubeconfig "\$KUBECONFIG_TENANT" apply --filename "\$BINDINGS/crbs\.yaml"\n\s*fi' documentIndex: 3 + + - it: a release with no users renders an empty bindings ConfigMap + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: [] + asserts: + - equal: + path: data["crbs.yaml"] + value: "" + - equal: + path: data["desired.txt"] + value: "" + + - it: CustomConfig mode renders the bindings ConfigMap for its users + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "false" + oidc.mode: CustomConfig + oidc.customConfig.config: | + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + jwt: + - issuer: + url: https://idp.example.com + audiences: [my-cluster] + oidc.users: + - email: alice@example.org + role: admin + asserts: + - isKind: + of: ConfigMap - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'name: cluster-admin' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'name: view' - documentIndex: 3 + path: data["crbs.yaml"] + pattern: 'name: alice@example.org' + + - it: mode=None renders no bindings ConfigMap even when users are set + # The Job takes its early branch at mode None and never reads the file. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + oidc.mode: None + oidc.users: + - email: alice@example.org + role: admin + asserts: + - hasDocuments: + count: 0 - it: mode=System aborts the Job instead of leaving the OIDC kubeconfig Secret as the install-time placeholder when the tenant CA is missing # A placeholder kubeconfig cannot authenticate, so the release must not go Ready with it. From 47b497194eab5187e09ecbebc1ca862ae229808d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 11:10:42 +0000 Subject: [PATCH 03/20] fix(backups): restore the ClickHouse archive a Backup was taken as The Altinity strategy named each archive inside its Pod with date -u and recorded nothing on the Backup, so a restore could only guess: it listed the remote archives under the application's prefix and took the newest. A RestoreJob of an older Backup therefore restored whichever backup had run last. The controller now names the archive before it creates the backup Job, from the BackupJob's persisted StartedAt, so every reconcile and every retried Pod of the Job writes the same one. The name is recorded in the Backup's driverMetadata next to the strategy parameters, and a restore renders it into the strategy Pod, which restores exactly that archive and fails, naming it, when remote storage no longer holds it. The name is recorded only when the Job's Pod was given it, so a Job created by an earlier controller, or a strategy that names its own archives, leaves the Backup without one instead of pointing it at an archive that does not exist. Because a retried Pod reuses the name, it waits on an attempt the sidecar is still running instead of being refused with 423, and asks for resume, so a local or partial remote backup an earlier attempt left is completed rather than refused as existing. Backups taken before names were recorded stay restorable: the Pod picks the newest archive whose name is stamped no later than the Backup's takenAt, which is set only after the backup Job finished, and logs a warning that the choice is a best effort. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- hack/altinity-backup-strategy.bats | 209 +++++++++++++++++ .../altinitystrategy_controller.go | 69 +++++- .../altinitystrategy_controller_test.go | 210 +++++++++++++++++- .../templates/strategy-altinity-default.yaml | 57 ++++- 4 files changed, 523 insertions(+), 22 deletions(-) create mode 100644 hack/altinity-backup-strategy.bats diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats new file mode 100644 index 000000000..234ef0b8b --- /dev/null +++ b/hack/altinity-backup-strategy.bats @@ -0,0 +1,209 @@ +#!/usr/bin/env bats +# Behavioural tests for the shipped Altinity (ClickHouse) backup/restore +# strategy script, extracted from the rendered cozy-default-altinity Strategy +# CR and EXECUTED against a stubbed clickhouse-backup HTTP API, with the real +# jq doing the archive selection. Which archive a restore reads back is +# decided by that selection over list/remote, so it is exercised against +# several archives rather than string-matched. +# +# The script runs with `bash`, not `sh`, for the reason given in +# hack/redis-backup-strategy.bats: it opens `set -euo pipefail`, which the +# production image's busybox ash supports and the CI runner's dash rejects. + +load test_helper + +CHART=packages/system/backupstrategy-controller + +# Render the shipped strategy's in-Pod script into $1 and substitute the +# controller-render placeholders the way the driver fills them: $2 mode, $3 +# app (release and source application), $4 the archive name recorded on the +# Backup (empty for a Backup that records none), $5 the Backup name, $6 its +# takenAt in the archive-name layout. An empty render fails early, as does a +# placeholder left unsubstituted. +render_script() { + out=$1; mode=$2; app=$3; archive=$4; backup=$5; takenat=$6 + helm template t "$CHART" -n cozy-backup-controller \ + --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ + | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ + | sed -e "s#{{ .Mode }}#${mode}#g" \ + -e "s#{{ .Release.Name }}#${app}#g" \ + -e "s#{{ .Release.Namespace }}#tenant-x#g" \ + -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \ + -e "s#{{ .Backup.Name }}#${backup}#g" \ + -e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \ + -e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out" + [ -s "$out" ] || return 1 + if grep -q '{{' "$out"; then + echo "rendered script still holds a controller placeholder" >&2 + return 1 + fi +} + +# Stub the clickhouse-backup API behind curl, first on PATH. list/remote +# serves $state/remote; actions serves $state/actions plus one success row per +# operation the script has issued, and after its first read swaps in +# $state/actions.next when a test seeds one (an operation that finishes while +# the script waits). Every request is logged to $state/curl.log. sleep is +# stubbed to fail after a few calls, so a script that never sees its +# operation finish fails the test instead of hanging it. +make_stubs() { + bin=$1; state=$2 + mkdir -p "$bin" + : > "$state/remote" + : > "$state/actions" + cat > "$bin/curl" <> "$state/curl.log" +case "\$method \$url" in + "GET "*/backup/status) echo '{}' ;; + "GET "*/backup/list/remote) cat "$state/remote" ;; + "GET "*/backup/actions) + cat "$state/actions" + if [ -f "$state/actions.done" ]; then cat "$state/actions.done"; fi + if [ -f "$state/actions.next" ]; then mv "$state/actions.next" "$state/actions"; fi + ;; + "POST "*/backup/create_remote*) + name=\${url#*name=}; name=\${name%%&*} + printf '{"command":"create_remote --resume %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\$name" >> "$state/actions.done" + ;; + "POST "*/backup/restore_remote/*) + printf '{"command":"restore_remote %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\${url##*/}" >> "$state/actions.done" + ;; + *) echo "unexpected request: \$method \$url" >&2; exit 22 ;; +esac +STUB + cat > "$bin/sleep" </dev/null || echo 0) +echo \$((n + 1)) > "$state/sleeps" +if [ "\$n" -ge 3 ]; then echo "sleep budget spent: the operation never finished" >&2; exit 1; fi +STUB + chmod 0755 "$bin/curl" "$bin/sleep" + export API_USERNAME=u API_PASSWORD=p +} + +# One list/remote row per argument, "name" or "name|desc"; created mirrors +# the name's timestamp, which is the order the pre-fix selection sorted by. +remote_rows() { + for row in "$@"; do + name=${row%%|*}; desc="tar, regular" + if [ "$row" != "$name" ]; then desc=${row#*|}; fi + ts=${name##*-} + printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc" + done +} + +@test "an empty render fails the fixture instead of passing as a no-op" { + tmp=$(mktemp -d) + CHART=$tmp/no-such-chart + if render_script "$tmp/s.sh" restore myapp "" bkp 20260304T050622Z 2>/dev/null; then + echo "FAIL: render_script reported success with no chart" + false + fi +} + +@test "restore reads back the archive the Backup recorded, not the newest one" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ + clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" + PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" + [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] +} + +@test "restore fails naming the recorded archive when remote storage lacks it" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/remote" + out="" + if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + echo "FAIL: restore succeeded without the recorded archive: $out" + false + fi + printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage' + if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then + echo "FAIL: restored another archive: $(cat "$tmp/curl.log")" + false + fi +} + +@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z + # Each decoy sorts after the expected archive and is ruled out by one + # filter: broken, not a bare timestamp after the prefix, or stamped after + # takenAt. + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ + "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ + clickhouse-myapp-20260304T050710Z-copy \ + clickhouse-myapp-20260304T060000Z > "$tmp/remote" + out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1) + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" + [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] + printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' +} + +@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z + remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" + out="" + if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + echo "FAIL: restore succeeded with every archive newer than takenAt: $out" + false + fi + printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z' + if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then + echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")" + false + fi +} + +@test "backup writes the archive under the name the controller chose" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" + PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" +} + +@test "a retried backup waits on the attempt the sidecar is still running" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" + printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"in progress","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions" + printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions.next" + PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" + # The sidecar answers a second operation with 423 while one runs. + if grep -qF 'POST ' "$tmp/curl.log"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")" + false + fi +} + +@test "a retried backup after a failed attempt resumes the same archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" + printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions" + PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" +} + +@test "backup falls back to a timestamped name when the controller supplies none" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp" + render_script "$tmp/s.sh" backup myapp "" "" "" + PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" + grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log" +} diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index b96d571f0..f7f60bde8 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -2,7 +2,9 @@ package backupcontroller import ( + "bytes" "context" + "encoding/json" "fmt" "maps" "strings" @@ -43,6 +45,18 @@ const ( // were in effect at backup time. altinityParamPrefix = "altinity.strategy.backups.cozystack.io/parameter/" + // altinityArchiveMetaKey records, on the Backup, the clickhouse-backup + // archive its Job wrote, as named by altinityArchiveName. A restore reads + // back exactly that archive: the newest one under the release's prefix + // may belong to a later Backup. + altinityArchiveMetaKey = "altinity.strategy.backups.cozystack.io/archive" + + // altinityArchiveTimeLayout is the UTC timestamp that ends an archive + // name. It is fixed-width, so names sharing a prefix sort chronologically + // as plain strings, which is what a restore of a Backup that records no + // archive relies on. + altinityArchiveTimeLayout = "20060102T150405Z" + // Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG / // Velero strategy cadence so behaviour is uniform across drivers. altinityPollInterval = 5 * time.Second @@ -102,14 +116,37 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { return out } +// altinityArchiveName mints the archive name for one BackupJob from its +// persisted StartedAt, so every reconcile and every Pod retry of the Job +// writes the same archive. It keeps the clickhouse-- +// shape, because a Backup that records no archive is restored by the +// timestamp in that shape. +func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { + return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) +} + +// altinityJobCarriesArchive reports whether the Job's Pod was rendered with +// the archive name. A Job created by a controller that predates archive +// names, or rendered from a strategy that names its own archives, wrote an +// archive under another name, and recording this one would point the Backup +// at an archive that does not exist. +func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool { + spec, err := json.Marshal(job.Spec.Template.Spec) + if err != nil { + return false + } + return bytes.Contains(spec, []byte(archive)) +} + // renderAltinityTemplate runs the strategy's PodTemplateSpec through the // repository's text/template engine with the same context shape as the CNPG // driver: every string field is templated against the application object, -// the release shorthand, the run mode, and the resolved parameters. +// the release shorthand, the run mode, the archive name, and the resolved +// parameters. func renderAltinityTemplate( tmpl corev1.PodTemplateSpec, app map[string]any, - releaseName, releaseNamespace, mode string, + releaseName, releaseNamespace, mode, archiveName string, parameters map[string]string, backup *backupsv1alpha1.Backup, ) (*corev1.PodTemplateSpec, error) { @@ -119,8 +156,9 @@ func renderAltinityTemplate( "Name": releaseName, "Namespace": releaseNamespace, }, - "Mode": mode, - "Parameters": parameters, + "Mode": mode, + "ArchiveName": archiveName, + "Parameters": parameters, } if backup != nil { // Expose ApplicationRef so restore strategies can reference the @@ -143,6 +181,9 @@ func renderAltinityTemplate( "Kind": backup.Spec.ApplicationRef.Kind, "Name": backup.Spec.ApplicationRef.Name, }, + // In the archive-name layout, so a strategy can compare it with + // the timestamp an archive name ends in. + "TakenAt": backup.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout), } } return template.Template(&tmpl, ctxMap) @@ -209,12 +250,14 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return ctrl.Result{}, err } + archive := altinityArchiveName(j) rendered, err := renderAltinityTemplate( strategy.Spec.Template, app, j.Spec.ApplicationRef.Name, j.Namespace, altinityModeBackup, + archive, resolved.Parameters, nil, ) @@ -239,7 +282,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved) + if !altinityJobCarriesArchive(batchJob, archive) { + archive = "" + } + artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive) if err != nil { return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) } @@ -411,16 +457,20 @@ func jobFailureMessage(j *batchv1.Job) string { } // createAltinityBackupArtifact materialises a Cozystack Backup carrying the -// strategy reference and the BackupClassStrategy parameters in effect at -// backup time. Parameters round-trip via DriverMetadata under -// altinityParamPrefix so a later RestoreJob can re-render the strategy -// template against the same values. +// strategy reference, the archive the Job wrote (when known) and the +// BackupClassStrategy parameters in effect at backup time. Parameters +// round-trip via DriverMetadata under altinityParamPrefix so a later +// RestoreJob can re-render the strategy template against the same values. func (r *BackupJobReconciler) createAltinityBackupArtifact( ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig, + archive string, ) (*backupsv1alpha1.Backup, error) { driverMD := map[string]string{} + if archive != "" { + driverMD[altinityArchiveMetaKey] = archive + } for k, v := range resolved.Parameters { driverMD[altinityParamPrefix+k] = v } @@ -616,6 +666,7 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res targetAppName, targetNamespace, altinityModeRestore, + backup.Spec.DriverMetadata[altinityArchiveMetaKey], backupParameters(backup), backup, ) diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 68f18fa28..7797842c0 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "testing" + "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -697,6 +698,7 @@ func TestBackupParameters_RoundTrip(t *testing.T) { DriverMetadata: map[string]string{ altinityParamPrefix + "bucketName": "b1", altinityParamPrefix + "endpoint": "https://s3", + altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z", "job.batch/name": "ignored", "": "ignored", }, @@ -706,7 +708,213 @@ func TestBackupParameters_RoundTrip(t *testing.T) { if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" { t.Errorf("backupParameters = %v", got) } - if _, ok := got["job.batch/name"]; ok { + if len(got) != 2 { t.Errorf("non-parameter key leaked through: %v", got) } } + +// newAltinityArchiveStrategy renders the archive name and the Backup's +// takenAt where a test can read them back off the Job. +func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { + s := newAltinityStrategy(name) + s.Spec.Template.Spec.Containers[0].Args = []string{ + "--mode={{ .Mode }}", + "--archive={{ .ArchiveName }}", + "--taken-at={{ .Backup.TakenAt }}", + "--bucket={{ .Parameters.bucketName }}", + } + return s +} + +func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) + j := &backupsv1alpha1.BackupJob{ + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + t.Errorf("altinityArchiveName = %q, want %q", got, want) + } +} + +// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob +// through Job creation and completion in separate reconciles. The archive +// name has to come out of the persisted StartedAt both times: the Pod is +// told it on the first, the Backup records it on the second, and a restore +// later reads back exactly that archive. +func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) + ctx := context.Background() + const want = "clickhouse-ch-test-20260304T050607Z" + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want { + t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want) + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want { + t.Errorf("Backup records archive %q, want %q", got, want) + } + if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" { + t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got) + } +} + +// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose +// Pod never carried the driver's archive name: one created before the +// upgrade that introduced the names, or rendered from a strategy that names +// its own archives. Recording the name would send the restore after an +// archive nobody wrote; recording none leaves it the fallback for Backups +// without a name. +func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace}, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "clickhouse-backup", + Image: "altinity/clickhouse-backup:test", + Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`}, + }}, + }}, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, + clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() error = %v", err) + } + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok { + t.Errorf("Backup records archive %q its Job was never given", got) + } + if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" { + t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got) + } +} + +func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) { + cases := []struct { + name string + archive string // recorded on the Backup; empty for a Backup taken before names were recorded + wantArchive string + }{ + {name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"}, + {name: "no recorded archive", archive: "", wantArchive: "--archive="}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + targetApp := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"} + if tc.archive != "" { + driverMD[altinityArchiveMetaKey] = tc.archive + } + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))), + DriverMetadata: driverMD, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + _, rr, _ := newAltinityTestEnv(t, targetApp, + clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob)) + ctx := context.Background() + + if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + job := &batchv1.Job{} + if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil { + t.Fatalf("get restore Job: %v", err) + } + args := job.Spec.Template.Spec.Containers[0].Args + want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"} + for i, w := range want { + if i >= len(args) || args[i] != w { + t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args) + } + } + }) + } +} diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 3397d6d0d..bb9917094 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -3,8 +3,8 @@ 1. Helm renders this file at chart install time. Anything inside `{{ ... }}` is interpreted by Helm. 2. The strategy controller renders the resulting Strategy CR - per-BackupJob, expanding `{{ .Application | .Release | .Mode }}` - against the live application object. + per-BackupJob, expanding `{{ .Application | .Release | .Mode | + .ArchiveName | .Backup }}` against the live application object. To emit a literal "{{ .Foo }}" that survives Helm, we pass it as the string argument of `printf` — Helm does not interpret inner template delimiters inside printf's quoted strings. Do NOT simplify by inlining @@ -51,23 +51,56 @@ spec: host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc" api="http://${host}:7171" auth="-u ${API_USERNAME}:${API_PASSWORD}" + # The archive the controller named for this BackupJob and records + # on its Backup. The default renders it empty, rather than as + # text/template's missing-key marker, when a controller that + # predates archive names renders this template during an upgrade. + archive="{{ printf `{{ .ArchiveName | default "" }}` }}" for _ in $(seq 1 60); do if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi sleep 5 done if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" - target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)" - echo "create_remote ${target}" - curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null + # Only that older controller leaves the name empty, and it + # records no name on the Backup either. + target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}" + # A retried Pod gets the same name. Wait on an attempt the + # sidecar is still running, which would answer a new one with + # 423, and ask for resume, so a local or partial remote backup + # an earlier attempt left is resumed rather than refused as + # existing, whatever use_resumable_state the sidecar runs with. + state=$(curl -fsS ${auth} "${api}/backup/actions" \ + | jq -rs --arg suffix "${target}" \ + '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last | .status // ""') + if [ "${state}" != "in progress" ]; then + echo "create_remote ${target}" + curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null + fi else - prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-" - target=$(curl -fsS ${auth} "${api}/backup/list/remote" \ - | jq -rs --arg prefix "${prefix}" \ - '[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""') - if [ -z "${target}" ] || [ "${target}" = "null" ]; then - echo "no remote backup found" >&2 - exit 1 + app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}" + backup="{{ printf "{{ .Backup.Name }}" }}" + prefix="clickhouse-${app}-" + remote=$(curl -fsS ${auth} "${api}/backup/list/remote") + if [ -n "${archive}" ]; then + if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then + echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2 + exit 1 + fi + target="${archive}" + else + # takenAt is set after the backup Job finished, so the + # archive that Job wrote is stamped no later than it. + notafter="{{ printf `{{ .Backup.TakenAt | default "" }}` }}" + echo "WARNING: Backup ${backup} records no archive name; restoring the newest archive of ${app} stamped no later than its takenAt (${notafter}), a best-effort choice that may not be the archive this Backup was taken as" >&2 + target=$(printf '%s\n' "${remote}" | jq -rs --arg prefix "${prefix}" --arg notafter "${notafter}" ' + [.[] | select((.desc // "") | startswith("broken") | not) | .name // "" + | select(startswith($prefix) and (ltrimstr($prefix) | test("^[0-9]{8}T[0-9]{6}Z$")) and ltrimstr($prefix) <= $notafter)] + | sort | last // ""') + if [ -z "${target}" ]; then + echo "no archive of ${app} stamped no later than ${notafter} in remote storage" >&2 + exit 1 + fi fi echo "restore_remote ${target}" curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null From 6e36385e00fcb9cc8b24e19ec8e9a5e11056b383 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Tue, 27 Oct 2026 18:25:19 +0300 Subject: [PATCH 04/20] feat(api): expose HelmRelease failure state on Application status Tenants usually read Applications but not the HelmReleases behind them, and only Ready and Released were copied over. A client could therefore see that a release was failing but not how often it had failed, nor whether helm-controller had stopped retrying it. Copy the Stalled condition, which helm-controller sets once it stops reconciling a release (remediation retries exhausted, rollback target missing, invalid CEL, access denied), and surface the install and upgrade failure counters with the last attempted release action. Stalled is dropped when it was observed for an older generation than the HelmRelease's current one: the Application carries no generation, so after a spec update a client would otherwise read the previous spec's failure as terminal for the new values. With the RetryOnFailure policy the platform generates today, a failed release is retried rather than stalled, so the counters are the signal. That does not hold for every state: a release missing from storage while an install failure count is retained still stalls, and a release on a remediation policy stalls once its retries run out. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- pkg/apis/apps/v1alpha1/types.go | 17 +++ .../apps/v1alpha1/applicationstatus.go | 38 ++++++ pkg/generated/openapi/zz_generated.openapi.go | 21 +++ pkg/registry/apps/application/rest.go | 14 +- .../application/rest_release_failure_test.go | 123 ++++++++++++++++++ 5 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 pkg/registry/apps/application/rest_release_failure_test.go diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go index 15bab146f..13270b0f1 100644 --- a/pkg/apis/apps/v1alpha1/types.go +++ b/pkg/apis/apps/v1alpha1/types.go @@ -55,6 +55,23 @@ type ApplicationStatus struct { // ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications. // +optional ExternalIPsCount int32 `json:"externalIPsCount,omitempty"` + // InstallFailures is the install failure count of the underlying HelmRelease. + // It counts only attempts that recorded a Helm release, so a failure before + // Helm records one, such as a chart that fails to render, shows only as + // Ready=False; a failure to apply the rendered resources is counted. It + // resets when the desired state (generation, chart version, values) changes, + // and on success only under the RetryOnFailure strategy, so a Ready release + // may carry it. + // +optional + InstallFailures int64 `json:"installFailures,omitempty"` + // UpgradeFailures is the upgrade failure count of the underlying HelmRelease, + // with the same counting and reset rules as InstallFailures. + // +optional + UpgradeFailures int64 `json:"upgradeFailures,omitempty"` + // LastAttemptedReleaseAction is the last release action (install or upgrade) + // the underlying HelmRelease attempted. + // +optional + LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"` } // SchedulingClass returns the scheduling class requested by this Application. diff --git a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go index e20e08a51..77db6b0e7 100644 --- a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go +++ b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go @@ -34,6 +34,20 @@ type ApplicationStatusApplyConfiguration struct { Namespace *string `json:"namespace,omitempty"` // ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications. ExternalIPsCount *int32 `json:"externalIPsCount,omitempty"` + // InstallFailures is the install failure count of the underlying HelmRelease. + // It counts only attempts that recorded a Helm release, so a failure before + // Helm records one, such as a chart that fails to render, shows only as + // Ready=False; a failure to apply the rendered resources is counted. It + // resets when the desired state (generation, chart version, values) changes, + // and on success only under the RetryOnFailure strategy, so a Ready release + // may carry it. + InstallFailures *int64 `json:"installFailures,omitempty"` + // UpgradeFailures is the upgrade failure count of the underlying HelmRelease, + // with the same counting and reset rules as InstallFailures. + UpgradeFailures *int64 `json:"upgradeFailures,omitempty"` + // LastAttemptedReleaseAction is the last release action (install or upgrade) + // the underlying HelmRelease attempted. + LastAttemptedReleaseAction *string `json:"lastAttemptedReleaseAction,omitempty"` } // ApplicationStatusApplyConfiguration constructs a declarative configuration of the ApplicationStatus type for use with @@ -78,3 +92,27 @@ func (b *ApplicationStatusApplyConfiguration) WithExternalIPsCount(value int32) b.ExternalIPsCount = &value return b } + +// WithInstallFailures sets the InstallFailures field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the InstallFailures field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithInstallFailures(value int64) *ApplicationStatusApplyConfiguration { + b.InstallFailures = &value + return b +} + +// WithUpgradeFailures sets the UpgradeFailures field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UpgradeFailures field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithUpgradeFailures(value int64) *ApplicationStatusApplyConfiguration { + b.UpgradeFailures = &value + return b +} + +// WithLastAttemptedReleaseAction sets the LastAttemptedReleaseAction field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the LastAttemptedReleaseAction field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithLastAttemptedReleaseAction(value string) *ApplicationStatusApplyConfiguration { + b.LastAttemptedReleaseAction = &value + return b +} diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 1edff5230..3c1e96f75 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -289,6 +289,27 @@ func schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref common.ReferenceCallbac Format: "int32", }, }, + "installFailures": { + SchemaProps: spec.SchemaProps{ + Description: "InstallFailures is the install failure count of the underlying HelmRelease. It counts only attempts that recorded a Helm release, so a failure before Helm records one, such as a chart that fails to render, shows only as Ready=False; a failure to apply the rendered resources is counted. It resets when the desired state (generation, chart version, values) changes, and on success only under the RetryOnFailure strategy, so a Ready release may carry it.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "upgradeFailures": { + SchemaProps: spec.SchemaProps{ + Description: "UpgradeFailures is the upgrade failure count of the underlying HelmRelease, with the same counting and reset rules as InstallFailures.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "lastAttemptedReleaseAction": { + SchemaProps: spec.SchemaProps{ + Description: "LastAttemptedReleaseAction is the last release action (install or upgrade) the underlying HelmRelease attempted.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 322ef6c6d..5f618b4e3 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H }, Spec: filteredSpec, Status: appsv1alpha1.ApplicationStatus{ - Version: hr.Status.LastAttemptedRevision, + Version: hr.Status.LastAttemptedRevision, + InstallFailures: hr.Status.InstallFailures, + UpgradeFailures: hr.Status.UpgradeFailures, + LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction), }, } var conditions []metav1.Condition for _, hrCondition := range hr.GetConditions() { - if hrCondition.Type == "Ready" || hrCondition.Type == "Released" { + // The Application carries no generation, so a client cannot tell a Stalled + // left over from the previous spec from one about the current spec. Compare + // the condition's own ObservedGeneration, not Status.ObservedGeneration: the + // question is which spec this condition describes. + if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation { + continue + } + if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" { conditions = append(conditions, metav1.Condition{ LastTransitionTime: hrCondition.LastTransitionTime, Reason: hrCondition.Reason, diff --git a/pkg/registry/apps/application/rest_release_failure_test.go b/pkg/registry/apps/application/rest_release_failure_test.go new file mode 100644 index 000000000..fc28f1553 --- /dev/null +++ b/pkg/registry/apps/application/rest_release_failure_test.go @@ -0,0 +1,123 @@ +package application + +import ( + "context" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Generation = 2 + hr.Status.Conditions = []metav1.Condition{ + {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2}, + {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2}, + {Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2}, + } + hr.Status.Failures = 7 + hr.Status.InstallFailures = 1 + hr.Status.UpgradeFailures = 3 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + sc := findCondition(app.GetConditions(), "Stalled") + if sc == nil { + t.Fatal("expected Stalled condition to be copied from the HelmRelease") + } + if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" { + t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason) + } + if app.Status.InstallFailures != 1 { + t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures) + } + if app.Status.UpgradeFailures != 3 { + t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures) + } + if app.Status.LastAttemptedReleaseAction != "install" { + t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction) + } +} + +// After a spec update the HelmRelease keeps the Stalled condition of the +// previous generation until helm-controller reconciles it; reported as is, it +// would read as a terminal failure of the new values. +func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Generation = 2 + hr.Status.Conditions = []metav1.Condition{ + {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1}, + {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1}, + } + hr.Status.InstallFailures = 1 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Stalled") != nil { + t.Error("expected Stalled observed at an older generation not to be copied") + } + if findCondition(app.GetConditions(), "Ready") == nil { + t.Error("expected Ready to still be copied") + } +} + +// helm-controller clears the counters on success only under the RetryOnFailure +// strategy, so a Ready release can still carry them; they must not be hidden. +func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"}, + {Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"}, + } + hr.Status.InstallFailures = 2 + hr.Status.UpgradeFailures = 1 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Stalled") != nil { + t.Error("expected no Stalled condition on a release that does not carry one") + } + if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 { + t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures) + } + if app.Status.LastAttemptedReleaseAction != "upgrade" { + t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction) + } +} + +func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"}, + {Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Reconciling") != nil { + t.Error("expected Reconciling not to be copied") + } + if findCondition(app.GetConditions(), "Ready") == nil { + t.Error("expected Ready to be copied") + } +} From d4ad5612d35d0272ec8a05ed0f90990102be7340 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 21 Oct 2026 10:53:05 +0400 Subject: [PATCH 05/20] fix(backups): report sizeBytes for MariaDB backup artifacts Backup.status.artifact.sizeBytes was only ever set by the etcd strategy, so MariaDB backups always arrived without a size. mariadb-operator writes each logical dump as /backup.[.].sql under a prefix shared by every backup of the application, and the URI the driver records is synthetic, so there is no single object to HEAD. On completion the driver now lists the prefix with the tenant credentials the operator Backup was rendered with and takes the size of the one dump whose timestamp falls inside the operator Backup's lifetime. When the bucket cannot be read or the window does not match exactly one dump, the field stays unset. The API doc now states that an absent sizeBytes means "unknown", and the Velero strategy documents why it leaves the field unset: kopia stores data content-addressed across backups, so there is no per-backup stored size. Refs #4316 Assisted-by: LLM Signed-off-by: Andrey Kolkov --- api/backups/v1alpha1/backup_types.go | 4 +- go.mod | 9 + go.sum | 19 ++ .../backupcontroller/backupjob_controller.go | 4 + .../backupcontroller/mariadb_artifact_size.go | 193 +++++++++++++++ .../mariadb_artifact_size_test.go | 234 ++++++++++++++++++ .../mariadbstrategy_controller.go | 5 +- .../velerostrategy_controller.go | 4 +- .../backups.cozystack.io_backups.yaml | 6 +- 9 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 internal/backupcontroller/mariadb_artifact_size.go create mode 100644 internal/backupcontroller/mariadb_artifact_size_test.go diff --git a/api/backups/v1alpha1/backup_types.go b/api/backups/v1alpha1/backup_types.go index fd76ac47e..483bb1fd8 100644 --- a/api/backups/v1alpha1/backup_types.go +++ b/api/backups/v1alpha1/backup_types.go @@ -37,7 +37,9 @@ type BackupArtifact struct { // For example: s3://bucket/prefix/file.tar.gz URI string `json:"uri"` - // SizeBytes is the size of the artifact in bytes, if known. + // SizeBytes is the size of the artifact in bytes, if known. Drivers + // leave it unset rather than writing 0 when they cannot measure the + // stored size, so an absent value means "unknown", not "empty". // +optional SizeBytes int64 `json:"sizeBytes,omitempty"` diff --git a/go.mod b/go.mod index 6109faf15..34352e583 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 github.com/prometheus/client_golang v1.24.1 @@ -63,6 +64,7 @@ require ( github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.4 // indirect github.com/gkampitakis/go-snaps v0.5.23 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v1.0.1 // indirect github.com/go-openapi/jsonreference v1.0.2 // indirect @@ -93,18 +95,25 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.71.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect github.com/tidwall/match v1.2.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/api/v3 v3.7.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect diff --git a/go.sum b/go.sum index dea6aaf08..667ca71ee 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg= github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -141,6 +143,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -151,6 +158,12 @@ github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -165,6 +178,8 @@ github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8 github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo= github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -180,6 +195,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= @@ -206,6 +223,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A= diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index c6b743356..f329eed4d 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -39,6 +39,10 @@ type BackupJobReconciler struct { // re-reads the BackupJob before dispatching to a driver. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to measure artifact + // sizes; nil means a real minio client. Set by unit tests. + newS3ObjectLister func(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) } func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go new file mode 100644 index 000000000..b968a98d4 --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSizeProbeTimeout = 30 * time.Second + + // Slack around the [created, completed] window of the operator Backup + // when matching the timestamp mariadb-operator embeds in the dump file + // name: that timestamp comes from the backup Job pod's clock, not the + // apiserver's, and is truncated to seconds. + mariadbSizeProbeClockSkew = time.Minute +) + +type s3ObjectInfo struct { + Key string + Size int64 +} + +type s3ObjectLister interface { + ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) +} + +// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for +// mdbBackup. The strategy prefix is shared by every backup of the +// application, so the dump is identified by the timestamp in its file name +// falling inside the operator Backup's lifetime. Returns 0 - leaving +// Artifact.SizeBytes absent - when the object store cannot be read or the +// window does not match exactly one dump: an unknown size must not be +// reported as a measured one. +func (r *BackupJobReconciler) mariadbArtifactSizeBytes( + ctx context.Context, + namespace string, + s3 *strategyv1alpha1.MariaDBS3Template, + mdbBackup *mariadbtypes.Backup, + completedAt time.Time, +) int64 { + logger := getLogger(ctx) + ctx, cancel := context.WithTimeout(ctx, mariadbSizeProbeTimeout) + defer cancel() + + newLister := r.newS3ObjectLister + if newLister == nil { + newLister = r.newMariaDBMinioLister + } + lister, err := newLister(ctx, namespace, s3) + if err != nil { + logger.Debug("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + return 0 + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + objects, err := lister.ListObjects(ctx, s3.Bucket, prefix) + if err != nil { + logger.Debug("cannot list MariaDB backup objects to measure size", "backup", mdbBackup.Name, "error", err) + return 0 + } + + lower := mdbBackup.CreationTimestamp.Add(-mariadbSizeProbeClockSkew) + upper := completedAt.Add(mariadbSizeProbeClockSkew) + var matched []s3ObjectInfo + for _, obj := range objects { + ts, ok := parseMariaDBLogicalBackupFile(strings.TrimPrefix(obj.Key, prefix)) + if !ok || ts.Before(lower) || ts.After(upper) { + continue + } + matched = append(matched, obj) + } + if len(matched) != 1 { + logger.Debug("MariaDB backup dump not attributable to a single object; leaving size unset", + "backup", mdbBackup.Name, "candidates", len(matched)) + return 0 + } + return matched[0].Size +} + +// mariadbOperatorS3Prefix mirrors how mariadb-operator normalises +// spec.storage.s3.prefix before prepending it to object keys. +func mariadbOperatorS3Prefix(prefix string) string { + if prefix == "" || prefix == "/" { + return "" + } + if !strings.HasSuffix(prefix, "/") { + return prefix + "/" + } + return prefix +} + +// parseMariaDBLogicalBackupFile accepts the logical-backup file names +// mariadb-operator writes, backup..sql and +// backup...sql, and returns the embedded timestamp. +func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) { + parts := strings.Split(name, ".") + if len(parts) != 3 && len(parts) != 4 { + return time.Time{}, false + } + if parts[0] != "backup" || parts[len(parts)-1] != "sql" { + return time.Time{}, false + } + ts, err := time.Parse(time.RFC3339, parts[1]) + if err != nil { + return time.Time{}, false + } + return ts, true +} + +type minioObjectLister struct { + client *minio.Client +} + +func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + var out []s3ObjectInfo + for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) { + if obj.Err != nil { + return nil, obj.Err + } + out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) + } + return out, nil +} + +// newMariaDBMinioLister builds an S3 client from the same Secrets and TLS +// settings the operator Backup was rendered with, so the controller reads +// the bucket with the tenant's credentials rather than its own. +func (r *BackupJobReconciler) newMariaDBMinioLister(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { + accessKey, err := r.mariadbSecretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef) + if err != nil { + return nil, err + } + secretKey, err := r.mariadbSecretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef) + if err != nil { + return nil, err + } + var sessionToken string + if s3.SessionTokenSecretKeyRef != nil { + if sessionToken, err = r.mariadbSecretValue(ctx, namespace, *s3.SessionTokenSecretKeyRef); err != nil { + return nil, err + } + } + + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, sessionToken), + Region: s3.Region, + Secure: s3.TLS != nil && s3.TLS.Enabled, + } + if opts.Secure && s3.TLS.CASecretKeyRef != nil { + caPEM, err := r.mariadbSecretValue(ctx, namespace, *s3.TLS.CASecretKeyRef) + if err != nil { + return nil, err + } + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM([]byte(caPEM)) { + return nil, fmt.Errorf("no PEM certificates in Secret %s/%s key %q", namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) + } + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}} + } + client, err := minio.New(s3.Endpoint, opts) + if err != nil { + return nil, err + } + return &minioObjectLister{client: client}, nil +} + +func (r *BackupJobReconciler) mariadbSecretValue(ctx context.Context, namespace string, ref strategyv1alpha1.MariaDBSecretKeySelector) (string, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: ref.Name}, secret); err != nil { + return "", err + } + v, ok := secret.Data[ref.Key] + if !ok { + return "", fmt.Errorf("Secret %s/%s has no key %q", namespace, ref.Name, ref.Key) + } + return string(v), nil +} diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go new file mode 100644 index 000000000..f7e450a26 --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +type fakeS3Lister struct { + objects []s3ObjectInfo + err error + + bucket, prefix string +} + +func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + f.bucket, f.prefix = bucket, prefix + return f.objects, f.err +} + +func reconcilerWithLister(l *fakeS3Lister) *BackupJobReconciler { + return &BackupJobReconciler{ + newS3ObjectLister: func(context.Context, string, *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { + return l, nil + }, + } +} + +func dumpKey(prefix string, ts time.Time, compression string) string { + if compression == "" { + return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339)) + } + return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) +} + +func TestParseMariaDBLogicalBackupFile(t *testing.T) { + cases := map[string]bool{ + "backup.2026-10-21T10:00:00Z.sql": true, + "backup.2026-10-21T10:00:00Z.gzip.sql": true, + "backup.2026-10-21T10:00:00+04:00.sql": true, + "backup.2026-10-21T10:00:00Z.sql.tmp": false, + "physicalbackup-20261021100000.xb": false, + "backup.not-a-date.sql": false, + "sub/backup.2026-10-21T10:00:00Z.sql": false, + "backup.2026-10-21T10:00:00Z.a.gzip.sql": false, + "backups.2026-10-21T10:00:00Z.gzip.sql": false, + "backup.2026-10-21T10:00:00Z.gzip.dump": false, + "": false, + } + for name, want := range cases { + if _, got := parseMariaDBLogicalBackupFile(name); got != want { + t.Errorf("parseMariaDBLogicalBackupFile(%q) = %v, want %v", name, got, want) + } + } +} + +func TestMariaDBOperatorS3Prefix(t *testing.T) { + cases := map[string]string{"": "", "/": "", "ns/app": "ns/app/", "ns/app/": "ns/app/"} + for in, want := range cases { + if got := mariadbOperatorS3Prefix(in); got != want { + t.Errorf("mariadbOperatorS3Prefix(%q) = %q, want %q", in, got, want) + } + } +} + +func TestMariaDBArtifactSizeBytes(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), + }} + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + const prefix = "tenant/db/" + + t.Run("single dump inside the backup window", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111}, + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242}, + {Key: prefix + "0-backup-target.txt", Size: 7}, + }} + got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) + if got != 4242 { + t.Errorf("size = %d, want 4242", got) + } + if l.bucket != "bkt" || l.prefix != prefix { + t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) + } + }) + + t.Run("pod clock skew within tolerance", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { + t.Errorf("size = %d, want 9", got) + } + }) + + t.Run("two dumps in the window leave size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1}, + {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0 (ambiguous)", got) + } + }) + + t.Run("no dump in the window leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1}, + }} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("list error leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("AccessDenied")} + if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { + c := newMariaDBStrategyTestClient(t) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) +} + +// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client +// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and +// response parsing are exercised together. +func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip") + + var gotPath, gotPrefix, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, ` + + bkttenant/db/11000false + %s31337"x"2026-10-21T10:01:00.000Z +`, key) + })) + defer srv.Close() + + creds := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"}, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("AKID"), + "AWS_SECRET_ACCESS_KEY": []byte("secret"), + }, + } + c := newMariaDBStrategyTestClient(t, creds) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Endpoint = strings.TrimPrefix(srv.URL, "http://") + s3.Prefix = "tenant/db/" + s3.Region = "us-east-1" + mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), + }} + + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, created.Add(3*time.Minute)); got != 31337 { + t.Fatalf("size = %d, want 31337", got) + } + if gotPath != "/bkt/" || gotPrefix != "tenant/db/" { + t.Errorf("request path=%q prefix=%q, want /bkt/ and tenant/db/", gotPath, gotPrefix) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } +} + +func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { + apps := mariadbapp.GroupName + completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) + mdbBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "src-op-bk", + CreationTimestamp: metav1.NewTime(completed.Add(-3 * time.Minute)), + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: completed, + }}}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + c := newMariaDBStrategyTestClient(t, job) + r := reconcilerWithLister(&fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048}, + }}) + r.Client, r.Scheme = c, c.Scheme() + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + + artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 { + t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index a5bd77094..84fe0c2df 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -392,7 +392,10 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( // operator's PVC ID), so an Artifact{URI: ""} is a worse signal than // no Artifact at all - leave it nil instead. if uri := driverMD[mariadbStorageURIKey]; uri != "" { - status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} + status.Artifact = &backupsv1alpha1.BackupArtifact{ + URI: uri, + SizeBytes: r.mariadbArtifactSizeBytes(ctx, j.Namespace, rendered.Storage.S3, mdbBackup, takenAt.Time), + } } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/backupcontroller/velerostrategy_controller.go b/internal/backupcontroller/velerostrategy_controller.go index e08b2f818..abedc97be 100644 --- a/internal/backupcontroller/velerostrategy_controller.go +++ b/internal/backupcontroller/velerostrategy_controller.go @@ -558,7 +558,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo veleroBackupNamespaceMetadataKey: veleroBackup.Namespace, } - // Create a basic artifact referencing the Velero backup + // SizeBytes stays unset: data lands in a content-addressed kopia + // repository shared across backups, so there is no per-backup stored + // size, and DataUpload totalBytes counts bytes read, not bytes stored. artifact := &backupsv1alpha1.BackupArtifact{ URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name), } diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index 28ef57fb7..f30d3dbc7 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -131,8 +131,10 @@ spec: For example: "sha256:". type: string sizeBytes: - description: SizeBytes is the size of the artifact in bytes, if - known. + description: |- + SizeBytes is the size of the artifact in bytes, if known. Drivers + leave it unset rather than writing 0 when they cannot measure the + stored size, so an absent value means "unknown", not "empty". format: int64 type: integer uri: From de60488a2cb9c25500af07ce13dbc467dfd72c98 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 21 Oct 2026 11:03:59 +0400 Subject: [PATCH 06/20] fix(backups): report sizeBytes for CNPG backup artifacts CNPG backups arrived without Backup.status.artifact.sizeBytes: cnpg.io/Backup status carries no size, and the data lives in barman's object-store layout rather than in a single object. barman-cloud uploads every base backup under //base// (backup.info plus the data and tablespace tarballs, possibly split into parts). On completion the driver now lists that prefix with the credentials and endpoint CA of the rendered barmanObjectStore and records the total. The WAL archive is not counted: it is one stream shared by every base backup of the server. The field stays unset when the backup ID is unknown, the bucket cannot be read, or nothing is stored under the prefix. The S3 client and Secret lookup are now shared with the MariaDB driver. Refs #4316 Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 102 ++++++++ .../backupcontroller/artifact_size_test.go | 49 ++++ .../backupcontroller/backupjob_controller.go | 2 +- .../backupcontroller/cnpg_artifact_size.go | 128 ++++++++++ .../cnpg_artifact_size_test.go | 229 ++++++++++++++++++ .../cnpgstrategy_controller.go | 3 +- .../backupcontroller/mariadb_artifact_size.go | 146 ++++------- .../mariadb_artifact_size_test.go | 49 +--- 8 files changed, 564 insertions(+), 144 deletions(-) create mode 100644 internal/backupcontroller/artifact_size.go create mode 100644 internal/backupcontroller/artifact_size_test.go create mode 100644 internal/backupcontroller/cnpg_artifact_size.go create mode 100644 internal/backupcontroller/cnpg_artifact_size_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go new file mode 100644 index 000000000..6ca2ad3ea --- /dev/null +++ b/internal/backupcontroller/artifact_size.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// artifactSizeProbeTimeout bounds the object-store round trips a driver +// makes to measure Backup.status.artifact.sizeBytes; the probe runs inside +// the reconcile that creates the Backup. +const artifactSizeProbeTimeout = 30 * time.Second + +type s3ObjectInfo struct { + Key string + Size int64 +} + +type s3ObjectLister interface { + ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) +} + +// s3Connection is what a driver resolves from its rendered strategy to +// read the bucket with the same identity the backup was written with. +type s3Connection struct { + // Endpoint is host[:port], without a scheme. + Endpoint string + Secure bool + Region string + AccessKey string + SecretKey string + SessionToken string + // CAPEM is trusted on top of the system pool when set. + CAPEM []byte +} + +func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { + if r.newS3ObjectLister != nil { + return r.newS3ObjectLister(conn) + } + return newMinioObjectLister(conn) +} + +type minioObjectLister struct { + client *minio.Client +} + +func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + opts := &minio.Options{ + Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), + Region: conn.Region, + Secure: conn.Secure, + } + if conn.Secure && len(conn.CAPEM) > 0 { + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(conn.CAPEM) { + return nil, errors.New("no PEM certificates in the endpoint CA bundle") + } + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}} + } + client, err := minio.New(conn.Endpoint, opts) + if err != nil { + return nil, err + } + return &minioObjectLister{client: client}, nil +} + +func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + var out []s3ObjectInfo + for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + return nil, obj.Err + } + out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) + } + return out, nil +} + +func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", err + } + v, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("Secret %s/%s has no key %q", namespace, name, key) + } + return string(v), nil +} diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go new file mode 100644 index 000000000..6e389cf66 --- /dev/null +++ b/internal/backupcontroller/artifact_size_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type fakeS3Lister struct { + objects []s3ObjectInfo + err error + + conn s3Connection + bucket, prefix string +} + +func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { + f.bucket, f.prefix = bucket, prefix + return f.objects, f.err +} + +func s3CredsSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("AKID"), + "AWS_SECRET_ACCESS_KEY": []byte("secret"), + }, + } +} + +// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed +// by a fake client seeded with objs (typically the credentials Secret). +func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler { + t.Helper() + c := newMariaDBStrategyTestClient(t, objs...) + return &BackupJobReconciler{ + Client: c, + Scheme: c.Scheme(), + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + }, + } +} diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index f329eed4d..b86278649 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -42,7 +42,7 @@ type BackupJobReconciler struct { // newS3ObjectLister overrides the S3 client used to measure artifact // sizes; nil means a real minio client. Set by unit tests. - newS3ObjectLister func(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/internal/backupcontroller/cnpg_artifact_size.go b/internal/backupcontroller/cnpg_artifact_size.go new file mode 100644 index 000000000..bff2d5b39 --- /dev/null +++ b/internal/backupcontroller/cnpg_artifact_size.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" +) + +// cnpgArtifactSizeBytes sums the objects barman-cloud uploaded for one base +// backup, //base//. The WAL archive +// is not counted: it is one continuous stream shared by every base backup of +// the server, so no part of it belongs to a single Backup. Returns 0 - +// leaving Artifact.SizeBytes absent - when the backup ID is unknown, the +// bucket cannot be read, or nothing is stored under the prefix. +func (r *BackupJobReconciler) cnpgArtifactSizeBytes( + ctx context.Context, + namespace string, + store *strategyv1alpha1.BarmanObjectStoreTemplate, + serverName, backupID string, +) int64 { + logger := getLogger(ctx) + if backupID == "" { + return 0 + } + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) + defer cancel() + + bucket, path, err := parseBarmanS3DestinationPath(store.DestinationPath) + if err != nil { + logger.Debug("cannot measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + conn, err := r.cnpgS3Connection(ctx, namespace, store) + if err != nil { + logger.Debug("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + lister, err := r.s3Lister(conn) + if err != nil { + logger.Debug("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + prefix := barmanBaseBackupPrefix(path, serverName, backupID) + objects, err := lister.ListObjects(ctx, bucket, prefix) + if err != nil { + logger.Debug("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err) + return 0 + } + var total int64 + for _, obj := range objects { + total += obj.Size + } + if total == 0 { + logger.Debug("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix) + } + return total +} + +func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace string, store *strategyv1alpha1.BarmanObjectStoreTemplate) (s3Connection, error) { + var conn s3Connection + if store.S3Credentials == nil { + return conn, errors.New("barmanObjectStore has no s3Credentials") + } + if store.EndpointURL == "" { + conn.Endpoint, conn.Secure = "s3.amazonaws.com", true + } else { + u, err := url.Parse(store.EndpointURL) + if err != nil || u.Host == "" { + return conn, fmt.Errorf("invalid endpointURL %q", store.EndpointURL) + } + conn.Endpoint, conn.Secure = u.Host, u.Scheme == "https" + } + + creds := store.S3Credentials + accessKeyKey, secretKeyKey := creds.AccessKeyIDKey, creds.SecretAccessKeyKey + if accessKeyKey == "" { + accessKeyKey = defaultS3AccessKeyIDKey + } + if secretKeyKey == "" { + secretKeyKey = defaultS3SecretAccessKeyKey + } + var err error + if conn.AccessKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, accessKeyKey); err != nil { + return conn, err + } + if conn.SecretKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, secretKeyKey); err != nil { + return conn, err + } + if ca := store.EndpointCA; ca != nil && conn.Secure { + key := ca.Key + if key == "" { + key = defaultEndpointCAKey + } + pem, err := r.secretValue(ctx, namespace, ca.SecretRef.Name, key) + if err != nil { + return conn, err + } + conn.CAPEM = []byte(pem) + } + return conn, nil +} + +// parseBarmanS3DestinationPath splits s3://bucket/path into the bucket and +// the key path barman-cloud prefixes every object with. +func parseBarmanS3DestinationPath(destinationPath string) (bucket, path string, err error) { + u, err := url.Parse(destinationPath) + if err != nil || u.Scheme != "s3" || u.Host == "" { + return "", "", fmt.Errorf("destinationPath %q is not an s3:// URL", destinationPath) + } + return u.Host, strings.Trim(u.Path, "/"), nil +} + +// barmanBaseBackupPrefix mirrors barman-cloud's +// os.path.join(path, server_name, "base", backup_id), with a trailing slash +// so one backup ID never matches another that extends it. +func barmanBaseBackupPrefix(path, serverName, backupID string) string { + parts := make([]string, 0, 4) + if path != "" { + parts = append(parts, path) + } + parts = append(parts, serverName, "base", backupID) + return strings.Join(parts, "/") + "/" +} diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go new file mode 100644 index 000000000..29f09253b --- /dev/null +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes" +) + +func newSizedBarmanObjectStore() *strategyv1alpha1.BarmanObjectStoreTemplate { + return &strategyv1alpha1.BarmanObjectStoreTemplate{ + DestinationPath: "s3://bkt/tenant/pg/", + EndpointURL: "https://s3.example:8443", + S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: "creds"}, + }, + } +} + +func TestBarmanBaseBackupPrefix(t *testing.T) { + cases := []struct{ path, want string }{ + {"tenant/pg", "tenant/pg/srv/base/20261021T100000/"}, + {"", "srv/base/20261021T100000/"}, + } + for _, tc := range cases { + if got := barmanBaseBackupPrefix(tc.path, "srv", "20261021T100000"); got != tc.want { + t.Errorf("barmanBaseBackupPrefix(%q) = %q, want %q", tc.path, got, tc.want) + } + } +} + +func TestParseBarmanS3DestinationPath(t *testing.T) { + cases := []struct { + in, bucket, path string + ok bool + }{ + {"s3://bkt/tenant/pg/", "bkt", "tenant/pg", true}, + {"s3://bkt", "bkt", "", true}, + {"s3://bkt/", "bkt", "", true}, + {"gs://bkt/x/", "", "", false}, + {"bkt/x/", "", "", false}, + } + for _, tc := range cases { + bucket, path, err := parseBarmanS3DestinationPath(tc.in) + if (err == nil) != tc.ok || bucket != tc.bucket || path != tc.path { + t.Errorf("parseBarmanS3DestinationPath(%q) = %q, %q, %v", tc.in, bucket, path, err) + } + } +} + +func TestCNPGArtifactSizeBytes(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + + t.Run("sums every object of the base backup", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: prefix + "backup.info", Size: 1}, + {Key: prefix + "data_0000.tar.gz", Size: 1000}, + {Key: prefix + "data_0001.tar.gz", Size: 200}, + {Key: prefix + "16385.tar.gz", Size: 30}, + }} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000") + if got != 1231 { + t.Errorf("size = %d, want 1231", got) + } + if l.bucket != "bkt" || l.prefix != prefix { + t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) + } + if l.conn.Endpoint != "s3.example:8443" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.SecretKey != "secret" { + t.Errorf("connection = %+v", l.conn) + } + }) + + t.Run("honours custom credential keys and the endpoint CA", func(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials.AccessKeyIDKey = "id" + store.S3Credentials.SecretAccessKeyKey = "key" + store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca"}} + creds := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"}, + Data: map[string][]byte{"id": []byte("ID2"), "key": []byte("KEY2")}, + } + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}} + r := reconcilerWithLister(t, l, creds, ca) + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 { + t.Errorf("size = %d, want 5", got) + } + if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" { + t.Errorf("connection = %+v", l.conn) + } + }) + + unset := map[string]struct { + store func() *strategyv1alpha1.BarmanObjectStoreTemplate + backupID string + lister *fakeS3Lister + noSecret bool + }{ + "unknown backup ID": { + store: newSizedBarmanObjectStore, lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "no objects under the prefix": { + store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{}, + }, + "list error": { + store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{err: errors.New("AccessDenied")}, + }, + "missing credentials Secret": { + store: newSizedBarmanObjectStore, backupID: "id", noSecret: true, + lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "no s3Credentials": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.S3Credentials = nil + return s + }, + backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "non-s3 destination": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.DestinationPath = "azure://container/x" + return s + }, + backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + } + for name, tc := range unset { + t.Run(name+" leaves size unset", func(t *testing.T) { + var r *BackupJobReconciler + if tc.noSecret { + r = reconcilerWithLister(t, tc.lister) + } else { + r = reconcilerWithLister(t, tc.lister, s3CredsSecret("tenant", "creds")) + } + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", tc.store(), "srv", tc.backupID); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + } +} + +// TestCNPGArtifactSizeBytes_MinioClient drives the real minio client against +// a stub ListObjectsV2 endpoint, so the endpoint URL parsing, Secret wiring +// and response parsing are exercised together. +func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + var gotPath, gotPrefix, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if _, ok := req.URL.Query()["location"]; ok { + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, ``) + return + } + gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, ` + + bkt%[1]s21000false + %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z + %[1]sdata.tar.gz4096"b"2026-10-21T10:01:00.000Z +`, prefix) + })) + defer srv.Close() + + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + store := newSizedBarmanObjectStore() + store.EndpointURL = srv.URL + + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 { + t.Fatalf("size = %d, want 4103", got) + } + if gotPath != "/bkt/" || gotPrefix != prefix { + t.Errorf("request path=%q prefix=%q, want /bkt/ and %q", gotPath, gotPrefix, prefix) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } +} + +func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"}, + } + cnpgBk := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, + Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}, + } + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 777}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 { + t.Fatalf("Artifact = %#v, want SizeBytes 777", got.Status.Artifact) + } + if l.prefix != "tenant/pg/srv/base/20261021T100000/" { + t.Errorf("listed prefix %q does not use the effective serverName", l.prefix) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 99e6b03f6..3b62ce6e1 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -855,7 +855,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( Status: backupsv1alpha1.BackupStatus{ Phase: backupsv1alpha1.BackupPhaseReady, Artifact: &backupsv1alpha1.BackupArtifact{ - URI: fmt.Sprintf("cnpg://%s/%s", serverName, cnpgBackup.Name), + URI: fmt.Sprintf("cnpg://%s/%s", serverName, cnpgBackup.Name), + SizeBytes: r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID), }, UnderlyingResources: underlyingResources, }, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index b968a98d4..20f68a454 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -3,40 +3,18 @@ package backupcontroller import ( "context" - "crypto/tls" - "crypto/x509" - "fmt" - "net/http" "strings" "time" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) -const ( - mariadbSizeProbeTimeout = 30 * time.Second - - // Slack around the [created, completed] window of the operator Backup - // when matching the timestamp mariadb-operator embeds in the dump file - // name: that timestamp comes from the backup Job pod's clock, not the - // apiserver's, and is truncated to seconds. - mariadbSizeProbeClockSkew = time.Minute -) - -type s3ObjectInfo struct { - Key string - Size int64 -} - -type s3ObjectLister interface { - ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) -} +// Slack around the [created, completed] window of the operator Backup when +// matching the timestamp mariadb-operator embeds in the dump file name: that +// timestamp comes from the backup Job pod's clock, not the apiserver's, and +// is truncated to seconds. +const mariadbSizeProbeClockSkew = time.Minute // mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for // mdbBackup. The strategy prefix is shared by every backup of the @@ -53,14 +31,15 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( completedAt time.Time, ) int64 { logger := getLogger(ctx) - ctx, cancel := context.WithTimeout(ctx, mariadbSizeProbeTimeout) + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() - newLister := r.newS3ObjectLister - if newLister == nil { - newLister = r.newMariaDBMinioLister + conn, err := r.mariadbS3Connection(ctx, namespace, s3) + if err != nil { + logger.Debug("cannot resolve S3 credentials to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + return 0 } - lister, err := newLister(ctx, namespace, s3) + lister, err := r.s3Lister(conn) if err != nil { logger.Debug("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) return 0 @@ -90,6 +69,37 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( return matched[0].Size } +// mariadbS3Connection resolves the Secrets and TLS settings the operator +// Backup was rendered with, so the bucket is read with the tenant's +// credentials rather than the controller's. +func (r *BackupJobReconciler) mariadbS3Connection(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3Connection, error) { + conn := s3Connection{ + Endpoint: s3.Endpoint, + Region: s3.Region, + Secure: s3.TLS != nil && s3.TLS.Enabled, + } + var err error + if conn.AccessKey, err = r.secretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef.Name, s3.AccessKeyIDSecretKeyRef.Key); err != nil { + return conn, err + } + if conn.SecretKey, err = r.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil { + return conn, err + } + if ref := s3.SessionTokenSecretKeyRef; ref != nil { + if conn.SessionToken, err = r.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil { + return conn, err + } + } + if conn.Secure && s3.TLS.CASecretKeyRef != nil { + ca, err := r.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) + if err != nil { + return conn, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil +} + // mariadbOperatorS3Prefix mirrors how mariadb-operator normalises // spec.storage.s3.prefix before prepending it to object keys. func mariadbOperatorS3Prefix(prefix string) string { @@ -119,75 +129,3 @@ func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) { } return ts, true } - -type minioObjectLister struct { - client *minio.Client -} - -func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { - var out []s3ObjectInfo - for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) { - if obj.Err != nil { - return nil, obj.Err - } - out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) - } - return out, nil -} - -// newMariaDBMinioLister builds an S3 client from the same Secrets and TLS -// settings the operator Backup was rendered with, so the controller reads -// the bucket with the tenant's credentials rather than its own. -func (r *BackupJobReconciler) newMariaDBMinioLister(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { - accessKey, err := r.mariadbSecretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef) - if err != nil { - return nil, err - } - secretKey, err := r.mariadbSecretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef) - if err != nil { - return nil, err - } - var sessionToken string - if s3.SessionTokenSecretKeyRef != nil { - if sessionToken, err = r.mariadbSecretValue(ctx, namespace, *s3.SessionTokenSecretKeyRef); err != nil { - return nil, err - } - } - - opts := &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, sessionToken), - Region: s3.Region, - Secure: s3.TLS != nil && s3.TLS.Enabled, - } - if opts.Secure && s3.TLS.CASecretKeyRef != nil { - caPEM, err := r.mariadbSecretValue(ctx, namespace, *s3.TLS.CASecretKeyRef) - if err != nil { - return nil, err - } - pool, err := x509.SystemCertPool() - if err != nil { - pool = x509.NewCertPool() - } - if !pool.AppendCertsFromPEM([]byte(caPEM)) { - return nil, fmt.Errorf("no PEM certificates in Secret %s/%s key %q", namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) - } - opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}} - } - client, err := minio.New(s3.Endpoint, opts) - if err != nil { - return nil, err - } - return &minioObjectLister{client: client}, nil -} - -func (r *BackupJobReconciler) mariadbSecretValue(ctx context.Context, namespace string, ref strategyv1alpha1.MariaDBSecretKeySelector) (string, error) { - secret := &corev1.Secret{} - if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: ref.Name}, secret); err != nil { - return "", err - } - v, ok := secret.Data[ref.Key] - if !ok { - return "", fmt.Errorf("Secret %s/%s has no key %q", namespace, ref.Name, ref.Key) - } - return string(v), nil -} diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index f7e450a26..214c7274c 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -14,32 +14,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) -type fakeS3Lister struct { - objects []s3ObjectInfo - err error - - bucket, prefix string -} - -func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { - f.bucket, f.prefix = bucket, prefix - return f.objects, f.err -} - -func reconcilerWithLister(l *fakeS3Lister) *BackupJobReconciler { - return &BackupJobReconciler{ - newS3ObjectLister: func(context.Context, string, *strategyv1alpha1.MariaDBS3Template) (s3ObjectLister, error) { - return l, nil - }, - } -} - func dumpKey(prefix string, ts time.Time, compression string) string { if compression == "" { return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339)) @@ -93,10 +72,13 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) + got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) if got != 4242 { t.Errorf("size = %d, want 4242", got) } + if l.conn.AccessKey != "AKID" || l.conn.Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the Secret's key and the rendered endpoint", l.conn) + } if l.bucket != "bkt" || l.prefix != prefix { t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) } @@ -106,7 +88,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9}, }} - if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -116,7 +98,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2}, }} - if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -125,14 +107,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1}, }} - if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(l).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -165,14 +147,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { })) defer srv.Close() - creds := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"}, - Data: map[string][]byte{ - "AWS_ACCESS_KEY_ID": []byte("AKID"), - "AWS_SECRET_ACCESS_KEY": []byte("secret"), - }, - } - c := newMariaDBStrategyTestClient(t, creds) + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} s3 := newRenderedMariaDBTemplate().Storage.S3 @@ -216,11 +191,9 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, }, } - c := newMariaDBStrategyTestClient(t, job) - r := reconcilerWithLister(&fakeS3Lister{objects: []s3ObjectInfo{ + r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048}, - }}) - r.Client, r.Scheme = c, c.Scheme() + }}, job, s3CredsSecret("tenant", "creds")) rendered := newRenderedMariaDBTemplate() rendered.Storage.S3.Prefix = "src/" From f222e52863ff2432329c3971c8a3d1882a6e379d Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 21 Oct 2026 12:12:11 +0400 Subject: [PATCH 07/20] fix(backups): harden the artifact size probe The probe runs inside the reconcile that creates the Backup, and the BackupJob controller has a single worker, so its timeout delays every other BackupJob while an endpoint hangs. Lower it from 30s to 10s. List the MariaDB prefix non-recursively, as mariadb-operator does. The default MariaDB prefix and the default CNPG destinationPath are the same // path, so a recursive listing of a MariaDB backed up next to a same-named Postgres walked the whole WAL archive and ran out of time. The CNPG base-backup prefix is still listed recursively. Build the custom-CA transport from minio.DefaultTransport so it keeps the proxy-from-environment and dial/TLS timeout settings a bare http.Transport drops, and cover it with a test against a TLS endpoint with a self-signed certificate. Log the reasons the size could not be measured (credentials, client, listing, empty CNPG prefix) at info. The chart runs the controller at info, which hid them before. The expected ambiguity of several MariaDB dumps in the window stays at debug. Skip an endpointCA without a Secret name, as buildBarmanObjectStore does. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 32 +++++---- .../backupcontroller/artifact_size_test.go | 5 +- .../backupcontroller/cnpg_artifact_size.go | 14 ++-- .../cnpg_artifact_size_test.go | 71 ++++++++++++++----- .../backupcontroller/mariadb_artifact_size.go | 11 +-- .../mariadb_artifact_size_test.go | 34 +++++++-- 6 files changed, 119 insertions(+), 48 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 6ca2ad3ea..0c6d52f19 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -3,11 +3,9 @@ package backupcontroller import ( "context" - "crypto/tls" "crypto/x509" "errors" "fmt" - "net/http" "time" "github.com/minio/minio-go/v7" @@ -17,9 +15,11 @@ import ( ) // artifactSizeProbeTimeout bounds the object-store round trips a driver -// makes to measure Backup.status.artifact.sizeBytes; the probe runs inside -// the reconcile that creates the Backup. -const artifactSizeProbeTimeout = 30 * time.Second +// makes to measure Backup.status.artifact.sizeBytes. The probe runs inside +// the reconcile that creates the Backup, and the BackupJob controller has a +// single worker, so an unreachable endpoint stalls every other BackupJob for +// this long. +const artifactSizeProbeTimeout = 10 * time.Second type s3ObjectInfo struct { Key string @@ -27,7 +27,7 @@ type s3ObjectInfo struct { } type s3ObjectLister interface { - ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) + ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) } // s3Connection is what a driver resolves from its rendered strategy to @@ -62,14 +62,22 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { Secure: conn.Secure, } if conn.Secure && len(conn.CAPEM) > 0 { - pool, err := x509.SystemCertPool() + // Start from minio's transport to keep its proxy and timeout settings. + tr, err := minio.DefaultTransport(true) if err != nil { - pool = x509.NewCertPool() + return nil, err + } + pool := tr.TLSClientConfig.RootCAs + if pool == nil { + if pool, err = x509.SystemCertPool(); err != nil { + pool = x509.NewCertPool() + } } if !pool.AppendCertsFromPEM(conn.CAPEM) { return nil, errors.New("no PEM certificates in the endpoint CA bundle") } - opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}} + tr.TLSClientConfig.RootCAs = pool + opts.Transport = tr } client, err := minio.New(conn.Endpoint, opts) if err != nil { @@ -78,9 +86,9 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { return &minioObjectLister{client: client}, nil } -func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { +func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { var out []s3ObjectInfo - for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}) { if obj.Err != nil { return nil, obj.Err } @@ -96,7 +104,7 @@ func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, } v, ok := secret.Data[key] if !ok { - return "", fmt.Errorf("Secret %s/%s has no key %q", namespace, name, key) + return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key) } return string(v), nil } diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 6e389cf66..c0a957852 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -16,10 +16,11 @@ type fakeS3Lister struct { conn s3Connection bucket, prefix string + recursive bool } -func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string) ([]s3ObjectInfo, error) { - f.bucket, f.prefix = bucket, prefix +func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + f.bucket, f.prefix, f.recursive = bucket, prefix, recursive return f.objects, f.err } diff --git a/internal/backupcontroller/cnpg_artifact_size.go b/internal/backupcontroller/cnpg_artifact_size.go index bff2d5b39..61b00d5e9 100644 --- a/internal/backupcontroller/cnpg_artifact_size.go +++ b/internal/backupcontroller/cnpg_artifact_size.go @@ -32,23 +32,23 @@ func (r *BackupJobReconciler) cnpgArtifactSizeBytes( bucket, path, err := parseBarmanS3DestinationPath(store.DestinationPath) if err != nil { - logger.Debug("cannot measure CNPG backup size", "backupID", backupID, "error", err) + logger.Info("cannot measure CNPG backup size", "backupID", backupID, "error", err) return 0 } conn, err := r.cnpgS3Connection(ctx, namespace, store) if err != nil { - logger.Debug("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err) + logger.Info("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err) return 0 } lister, err := r.s3Lister(conn) if err != nil { - logger.Debug("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err) + logger.Info("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err) return 0 } prefix := barmanBaseBackupPrefix(path, serverName, backupID) - objects, err := lister.ListObjects(ctx, bucket, prefix) + objects, err := lister.ListObjects(ctx, bucket, prefix, true) if err != nil { - logger.Debug("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err) + logger.Info("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err) return 0 } var total int64 @@ -56,7 +56,7 @@ func (r *BackupJobReconciler) cnpgArtifactSizeBytes( total += obj.Size } if total == 0 { - logger.Debug("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix) + logger.Info("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix) } return total } @@ -91,7 +91,7 @@ func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace st if conn.SecretKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, secretKeyKey); err != nil { return conn, err } - if ca := store.EndpointCA; ca != nil && conn.Secure { + if ca := store.EndpointCA; ca != nil && ca.SecretRef.Name != "" && conn.Secure { key := ca.Key if key == "" { key = defaultEndpointCAKey diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 29f09253b..81d850f64 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -3,6 +3,7 @@ package backupcontroller import ( "context" + "encoding/pem" "errors" "fmt" "net/http" @@ -74,8 +75,8 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { if got != 1231 { t.Errorf("size = %d, want 1231", got) } - if l.bucket != "bkt" || l.prefix != prefix { - t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) + if l.bucket != "bkt" || l.prefix != prefix || !l.recursive { + t.Errorf("listed %q/%q recursive=%v, want bkt/%q recursive", l.bucket, l.prefix, l.recursive, prefix) } if l.conn.Endpoint != "s3.example:8443" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.SecretKey != "secret" { t.Errorf("connection = %+v", l.conn) @@ -161,21 +162,10 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { // and response parsing are exercised together. func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) { const prefix = "tenant/pg/srv/base/20261021T100000/" - var gotPath, gotPrefix, gotAuth string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if _, ok := req.URL.Query()["location"]; ok { - w.Header().Set("Content-Type", "application/xml") - fmt.Fprint(w, ``) - return - } + var gotPath, gotPrefix, gotDelimiter, gotAuth string + srv := httptest.NewServer(cnpgListStub(prefix, func(req *http.Request) { gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") - w.Header().Set("Content-Type", "application/xml") - fmt.Fprintf(w, ` - - bkt%[1]s21000false - %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z - %[1]sdata.tar.gz4096"b"2026-10-21T10:01:00.000Z -`, prefix) + gotDelimiter = req.URL.Query().Get("delimiter") })) defer srv.Close() @@ -187,14 +177,59 @@ func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) { if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 { t.Fatalf("size = %d, want 4103", got) } - if gotPath != "/bkt/" || gotPrefix != prefix { - t.Errorf("request path=%q prefix=%q, want /bkt/ and %q", gotPath, gotPrefix, prefix) + if gotPath != "/bkt/" || gotPrefix != prefix || gotDelimiter != "" { + t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, %q and a recursive listing", gotPath, gotPrefix, gotDelimiter, prefix) } if !strings.Contains(gotAuth, "Credential=AKID/") { t.Errorf("request not signed with the Secret's access key: %q", gotAuth) } } +func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/xml") + if _, ok := req.URL.Query()["location"]; ok { + fmt.Fprint(w, ``) + return + } + if seen != nil { + seen(req) + } + fmt.Fprintf(w, ` + + bkt%[1]s21000false + %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z + %[1]sdata.tar.gz4096"b"2026-10-21T10:01:00.000Z +`, prefix) + } +} + +// TestCNPGArtifactSizeBytes_EndpointCA serves the listing over TLS with a +// self-signed certificate: the probe must trust it only through endpointCA. +func TestCNPGArtifactSizeBytes_EndpointCA(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + srv := httptest.NewTLSServer(cnpgListStub(prefix, nil)) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-ca"}, + Data: map[string][]byte{"ca.crt": caPEM}, + } + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"), ca) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + store := newSizedBarmanObjectStore() + store.EndpointURL = srv.URL + + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 0 { + t.Errorf("without endpointCA: size = %d, want 0 (untrusted certificate)", got) + } + store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "s3-ca"}} + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 { + t.Errorf("with endpointCA: size = %d, want 4103", got) + } +} + func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup strategyGroup := strategyv1alpha1.GroupVersion.Group diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 20f68a454..b13e7ff1f 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -36,18 +36,21 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( conn, err := r.mariadbS3Connection(ctx, namespace, s3) if err != nil { - logger.Debug("cannot resolve S3 credentials to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + logger.Info("cannot resolve S3 credentials to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) return 0 } lister, err := r.s3Lister(conn) if err != nil { - logger.Debug("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + logger.Info("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) return 0 } + // Non-recursive, as mariadb-operator lists it: the default prefix is also + // the default CNPG destinationPath, so an app of each kind with the same + // name shares it and a recursive listing would walk the WAL archive. prefix := mariadbOperatorS3Prefix(s3.Prefix) - objects, err := lister.ListObjects(ctx, s3.Bucket, prefix) + objects, err := lister.ListObjects(ctx, s3.Bucket, prefix, false) if err != nil { - logger.Debug("cannot list MariaDB backup objects to measure size", "backup", mdbBackup.Name, "error", err) + logger.Info("cannot list MariaDB backup objects to measure size", "backup", mdbBackup.Name, "error", err) return 0 } diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 214c7274c..0260cb14e 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -14,6 +14,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" @@ -79,8 +80,29 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { if l.conn.AccessKey != "AKID" || l.conn.Endpoint != "s3.example:9000" { t.Errorf("connection = %+v, want the Secret's key and the rendered endpoint", l.conn) } - if l.bucket != "bkt" || l.prefix != prefix { - t.Errorf("listed %q/%q, want bkt/%q", l.bucket, l.prefix, prefix) + if l.bucket != "bkt" || l.prefix != prefix || l.recursive { + t.Errorf("listed %q/%q recursive=%v, want bkt/%q non-recursive", l.bucket, l.prefix, l.recursive, prefix) + } + }) + + t.Run("reads the endpoint CA from its Secret", func(t *testing.T) { + tlsS3 := newRenderedMariaDBTemplate().Storage.S3 + tlsS3.Prefix = "tenant/db" + tlsS3.TLS = &strategyv1alpha1.MariaDBS3TLS{ + Enabled: true, + CASecretKeyRef: &strategyv1alpha1.MariaDBSecretKeySelector{Name: "ca", Key: "ca.crt"}, + } + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", tlsS3, mdbBackup, completed); got != 3 { + t.Errorf("size = %d, want 3", got) + } + if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { + t.Errorf("connection = %+v, want TLS with the Secret's CA", l.conn) } }) @@ -135,14 +157,16 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip") - var gotPath, gotPrefix, gotAuth string + var gotPath, gotPrefix, gotDelimiter, gotAuth string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + gotDelimiter = req.URL.Query().Get("delimiter") w.Header().Set("Content-Type", "application/xml") fmt.Fprintf(w, ` bkttenant/db/11000false %s31337"x"2026-10-21T10:01:00.000Z + tenant/db/srv/ `, key) })) defer srv.Close() @@ -161,8 +185,8 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, created.Add(3*time.Minute)); got != 31337 { t.Fatalf("size = %d, want 31337", got) } - if gotPath != "/bkt/" || gotPrefix != "tenant/db/" { - t.Errorf("request path=%q prefix=%q, want /bkt/ and tenant/db/", gotPath, gotPrefix) + if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { + t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, tenant/db/ and a non-recursive listing", gotPath, gotPrefix, gotDelimiter) } if !strings.Contains(gotAuth, "Credential=AKID/") { t.Errorf("request not signed with the Secret's access key: %q", gotAuth) From 038f35376ab269d75e18cb6e56450bd7e616b06b Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 21 Oct 2026 12:44:23 +0400 Subject: [PATCH 08/20] fix(backups): do not report a partial listing or re-probe on retry minio-go ends a listing whose context is cancelled between pages by closing the channel without an error item. When the probe timeout expired between pages of a large CNPG base backup, the pages already read were summed and reported as the measured size. Check the context once the listing channel closes and leave the size unset instead. The probe ran before the Backup was created, so a reconcile that retried after the Backup already existed probed the bucket again, held the single BackupJob worker for up to the timeout, and then discarded the result in favour of the existing object. Return an existing Backup before probing. Log the ambiguous MariaDB case (several dumps in the window) at info like the other reasons the size stays unset, and cover the session-token credential path with a test. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 30 ++++++++++- .../backupcontroller/artifact_size_test.go | 32 ++++++++++++ .../cnpg_artifact_size_test.go | 25 ++++++++- .../cnpgstrategy_controller.go | 3 ++ .../backupcontroller/mariadb_artifact_size.go | 2 +- .../mariadb_artifact_size_test.go | 51 +++++++++++++++++++ .../mariadbstrategy_controller.go | 3 ++ 7 files changed, 143 insertions(+), 3 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 0c6d52f19..6d65284a7 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -11,7 +11,10 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) // artifactSizeProbeTimeout bounds the object-store round trips a driver @@ -87,16 +90,41 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { } func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) +} + +// collectObjects drains a minio listing. minio-go ends a listing whose +// context is cancelled between pages by closing the channel without an +// error item, so a cancelled context is checked once the channel closes: +// otherwise a timeout would pass a partial listing off as complete. +func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3ObjectInfo, error) { var out []s3ObjectInfo - for obj := range l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}) { + for obj := range objects { if obj.Err != nil { return nil, obj.Err } out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) } + if err := ctx.Err(); err != nil { + return nil, err + } return out, nil } +// existingBackup returns the Backup a previous reconcile already created +// for the job, so a retry skips the size probe whose result it would +// discard. Returns (nil, nil) when there is none. +func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (*backupsv1alpha1.Backup, error) { + existing := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return existing, nil +} + func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index c0a957852..cfe95d16d 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -3,8 +3,11 @@ package backupcontroller import ( "context" + "errors" "testing" + "github.com/minio/minio-go/v7" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,9 +20,11 @@ type fakeS3Lister struct { conn s3Connection bucket, prefix string recursive bool + calls int } func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + f.calls++ f.bucket, f.prefix, f.recursive = bucket, prefix, recursive return f.objects, f.err } @@ -48,3 +53,30 @@ func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) }, } } + +func TestCollectObjects(t *testing.T) { + listing := func() <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo, 1) + ch <- minio.ObjectInfo{Key: "a", Size: 1} + close(ch) + return ch + } + + got, err := collectObjects(context.Background(), listing()) + if err != nil || len(got) != 1 || got[0].Size != 1 { + t.Fatalf("collectObjects = %v, %v; want one object", got, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) { + t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err) + } + + errCh := make(chan minio.ObjectInfo, 1) + errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")} + close(errCh) + if _, err := collectObjects(context.Background(), errCh); err == nil { + t.Error("listing error was not returned") + } +} diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 81d850f64..2db8bc084 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -248,7 +248,7 @@ func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { } rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 777}}} - r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + r := reconcilerWithLister(t, l) r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) @@ -262,3 +262,26 @@ func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { t.Errorf("listed prefix %q does not use the effective serverName", l.prefix) } } + +func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, existing, s3CredsSecret("tenant", "creds")) + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + + if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that already exists", l.calls) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 3b62ce6e1..50e9dd8ab 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -815,6 +815,9 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { + if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil { + return existing, err + } takenAt := metav1.Now() if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() { takenAt = *cnpgBackup.Status.StartedAt diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index b13e7ff1f..86d31694a 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -65,7 +65,7 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( matched = append(matched, obj) } if len(matched) != 1 { - logger.Debug("MariaDB backup dump not attributable to a single object; leaving size unset", + logger.Info("MariaDB backup dump not attributable to a single object; leaving size unset", "backup", mdbBackup.Name, "candidates", len(matched)) return 0 } diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 0260cb14e..d068d9898 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -106,6 +106,24 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } }) + t.Run("reads the session token from its Secret", func(t *testing.T) { + stsS3 := newRenderedMariaDBTemplate().Storage.S3 + stsS3.Prefix = "tenant/db" + stsS3.SessionTokenSecretKeyRef = &strategyv1alpha1.MariaDBSecretKeySelector{Name: "sts", Key: "token"} + sts := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "sts"}, + Data: map[string][]byte{"token": []byte("TOKEN")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", stsS3, mdbBackup, completed); got != 4 { + t.Errorf("size = %d, want 4", got) + } + if l.conn.SessionToken != "TOKEN" { + t.Errorf("session token = %q, want the Secret's", l.conn.SessionToken) + } + }) + t.Run("pod clock skew within tolerance", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9}, @@ -229,3 +247,36 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) } } + +func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + existing := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Status: backupsv1alpha1.BackupStatus{Artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bkt/src/op-bk", SizeBytes: 99}}, + } + l := &fakeS3Lister{} + r := reconcilerWithLister(t, l, job, existing, s3CredsSecret("tenant", "creds")) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + + got, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that already exists", l.calls) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 99 { + t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 84fe0c2df..bdffc7e7c 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -365,6 +365,9 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( mdbBackup *mariadbtypes.Backup, rendered *strategyv1alpha1.MariaDBTemplate, ) (*backupsv1alpha1.Backup, error) { + if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil { + return existing, err + } takenAt := metav1.Now() if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() { takenAt = cond.LastTransitionTime From 3b86f578035006b392b672d90d8662644d910991 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 21 Oct 2026 20:12:38 +0400 Subject: [PATCH 09/20] fix(backups): refuse to probe an S3 endpoint behind a path prefix An endpoint such as https://s3.example.org/prefix cannot be addressed by minio-go, which has no path-prefix option, and the prefix cannot be added below the client either, because the SigV4 signature covers the request path. The CNPG probe was worse than failing: it kept only the host of endpointURL and listed the bucket at a location barman never writes to. Reject such endpoints with an explicit error, in the CNPG probe and in the shared client, so the size is left unset instead of measured somewhere else. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 9 ++++++++- internal/backupcontroller/artifact_size_test.go | 9 +++++++++ internal/backupcontroller/cnpg_artifact_size.go | 5 +++++ internal/backupcontroller/cnpg_artifact_size_test.go | 8 ++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 6d65284a7..7a5fec8f8 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "errors" "fmt" + "strings" "time" "github.com/minio/minio-go/v7" @@ -36,7 +37,7 @@ type s3ObjectLister interface { // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { - // Endpoint is host[:port], without a scheme. + // Endpoint is host[:port], without a scheme or a path. Endpoint string Secure bool Region string @@ -59,6 +60,12 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + // An endpoint behind a path prefix cannot be addressed: minio-go has no + // such option, and the SigV4 signature covers the request path, so the + // prefix cannot be added below the client either. + if strings.Contains(conn.Endpoint, "/") { + return nil, fmt.Errorf("endpoint %q has a path, which the S3 client cannot address", conn.Endpoint) + } opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index cfe95d16d..5b636a44b 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -80,3 +80,12 @@ func TestCollectObjects(t *testing.T) { t.Error("listing error was not returned") } } + +func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) { + if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org/prefix", Secure: true}); err == nil { + t.Error("an endpoint with a path was accepted") + } + if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org:9000", Secure: true}); err != nil { + t.Errorf("host:port endpoint rejected: %v", err) + } +} diff --git a/internal/backupcontroller/cnpg_artifact_size.go b/internal/backupcontroller/cnpg_artifact_size.go index 61b00d5e9..c7d91bdb2 100644 --- a/internal/backupcontroller/cnpg_artifact_size.go +++ b/internal/backupcontroller/cnpg_artifact_size.go @@ -73,6 +73,11 @@ func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace st if err != nil || u.Host == "" { return conn, fmt.Errorf("invalid endpointURL %q", store.EndpointURL) } + // Dropping the path would probe the bucket at another location than + // the one barman writes to. + if strings.Trim(u.Path, "/") != "" { + return conn, fmt.Errorf("endpointURL %q has a path, which the S3 client cannot address", store.EndpointURL) + } conn.Endpoint, conn.Secure = u.Host, u.Scheme == "https" } diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 2db8bc084..de3960261 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -141,6 +141,14 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { }, backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, }, + "endpointURL with a path": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.EndpointURL = "https://s3.example:8443/prefix" + return s + }, + backupID: "20261021T100000", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, } for name, tc := range unset { t.Run(name+" leaves size unset", func(t *testing.T) { From a3d854739d17602575ed6cb2ffa1a885c0892d01 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 22 Oct 2026 17:30:32 +0400 Subject: [PATCH 10/20] fix(backups): attribute MariaDB dumps on two clocks and read retries uncached The MariaDB probe identified this run's dump by the timestamp in its file name, which comes from the backup pod's clock. With that clock skewed past the slack, this run's dump falls outside the window while a neighbouring run's dump can be the single match left, and its size is reported as this backup's. A dump now has to fall inside the window by both its name and its LastModified, which the object store stamps, so a single skewed clock leaves the size unset instead. The name parser also accepts the layout later mariadb-operator releases write for compressed dumps, backup..sql., and checks the compression token in either layout, so an operator upgrade does not silently stop the measurement. The retry check and the AlreadyExists refetch read the Backup through the cache. A retry usually follows a Create made moments earlier; a cache that does not show it yet turned the refetch into NotFound and failed a backup that had succeeded. Read it from the API server instead. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 13 ++++--- .../backupcontroller/artifact_size_test.go | 17 +++++++++ .../cnpg_artifact_size_test.go | 13 +++++++ .../cnpgstrategy_controller.go | 2 +- .../backupcontroller/mariadb_artifact_size.go | 37 ++++++++++++++----- .../mariadb_artifact_size_test.go | 36 +++++++++++++----- .../mariadbstrategy_controller.go | 2 +- 7 files changed, 94 insertions(+), 26 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 7a5fec8f8..517fe07a6 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -26,8 +26,9 @@ import ( const artifactSizeProbeTimeout = 10 * time.Second type s3ObjectInfo struct { - Key string - Size int64 + Key string + Size int64 + LastModified time.Time } type s3ObjectLister interface { @@ -110,7 +111,7 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O if obj.Err != nil { return nil, obj.Err } - out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size}) + out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size, LastModified: obj.LastModified}) } if err := ctx.Err(); err != nil { return nil, err @@ -120,10 +121,12 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O // existingBackup returns the Backup a previous reconcile already created // for the job, so a retry skips the size probe whose result it would -// discard. Returns (nil, nil) when there is none. +// discard. Returns (nil, nil) when there is none. It reads past the cache: +// a retry typically follows a Create made moments earlier, which the cache +// may not show yet. func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (*backupsv1alpha1.Backup, error) { existing := &backupsv1alpha1.Backup{} - if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { + if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { if apierrors.IsNotFound(err) { return nil, nil } diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 5b636a44b..f24079f8b 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -11,6 +11,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) type fakeS3Lister struct { @@ -89,3 +91,18 @@ func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) { t.Errorf("host:port endpoint rejected: %v", err) } } + +// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a +// Backup the cache does not show yet: the previous reconcile created it +// moments earlier. +func TestExistingBackup_ReadsPastTheCache(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}} + r := &BackupJobReconciler{ + Client: newMariaDBStrategyTestClient(t), + APIReader: newMariaDBStrategyTestClient(t, backup), + } + got, err := r.existingBackup(context.Background(), "tenant", "bj") + if err != nil || got == nil { + t.Fatalf("existingBackup = %v, %v; want the Backup the API server holds", got, err) + } +} diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index de3960261..13b647bdd 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -83,6 +83,19 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { } }) + t.Run("an empty endpointURL means AWS", func(t *testing.T) { + store := newSizedBarmanObjectStore() + store.EndpointURL = "" + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 { + t.Errorf("size = %d, want 5", got) + } + if l.conn.Endpoint != "s3.amazonaws.com" || !l.conn.Secure { + t.Errorf("connection = %+v, want s3.amazonaws.com over TLS", l.conn) + } + }) + t.Run("honours custom credential keys and the endpoint CA", func(t *testing.T) { store := newSizedBarmanObjectStore() store.S3Credentials.AccessKeyIDKey = "id" diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 50e9dd8ab..5dabfa66c 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -876,7 +876,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { return nil, getErr } return existing, nil diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 86d31694a..273eb8105 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -11,15 +11,17 @@ import ( ) // Slack around the [created, completed] window of the operator Backup when -// matching the timestamp mariadb-operator embeds in the dump file name: that -// timestamp comes from the backup Job pod's clock, not the apiserver's, and -// is truncated to seconds. +// matching a dump: the timestamp in its file name comes from the backup Job +// pod's clock and is truncated to seconds, and its LastModified comes from +// the object store's clock, neither of which is the apiserver's. const mariadbSizeProbeClockSkew = time.Minute // mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for // mdbBackup. The strategy prefix is shared by every backup of the -// application, so the dump is identified by the timestamp in its file name -// falling inside the operator Backup's lifetime. Returns 0 - leaving +// application, so the dump is identified by both the timestamp in its file +// name and its LastModified falling inside the operator Backup's lifetime: +// the two come from different clocks, so a skewed pod clock alone cannot +// pull a neighbouring run's dump into the window. Returns 0 - leaving // Artifact.SizeBytes absent - when the object store cannot be read or the // window does not match exactly one dump: an unknown size must not be // reported as a measured one. @@ -59,7 +61,7 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( var matched []s3ObjectInfo for _, obj := range objects { ts, ok := parseMariaDBLogicalBackupFile(strings.TrimPrefix(obj.Key, prefix)) - if !ok || ts.Before(lower) || ts.After(upper) { + if !ok || !inWindow(ts, lower, upper) || !inWindow(obj.LastModified, lower, upper) { continue } matched = append(matched, obj) @@ -115,15 +117,30 @@ func mariadbOperatorS3Prefix(prefix string) string { return prefix } +func inWindow(t, lower, upper time.Time) bool { + return !t.Before(lower) && !t.After(upper) +} + +// Compression tokens mariadb-operator puts in a dump file name: the +// algorithm before .sql in v25.10 (backup..gzip.sql), the extension +// after it in later releases (backup..sql.gz). +var ( + mariadbDumpAlgorithms = map[string]bool{"none": true, "bzip2": true, "gzip": true, "zstd": true} + mariadbDumpExtensions = map[string]bool{"bz2": true, "gz": true, "zst": true} +) + // parseMariaDBLogicalBackupFile accepts the logical-backup file names -// mariadb-operator writes, backup..sql and -// backup...sql, and returns the embedded timestamp. +// mariadb-operator writes and returns the embedded timestamp. func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) { parts := strings.Split(name, ".") - if len(parts) != 3 && len(parts) != 4 { + if parts[0] != "backup" { return time.Time{}, false } - if parts[0] != "backup" || parts[len(parts)-1] != "sql" { + switch { + case len(parts) == 3 && parts[2] == "sql": + case len(parts) == 4 && parts[2] == "sql" && mariadbDumpExtensions[parts[3]]: + case len(parts) == 4 && parts[3] == "sql" && mariadbDumpAlgorithms[parts[2]]: + default: return time.Time{}, false } ts, err := time.Parse(time.RFC3339, parts[1]) diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index d068d9898..6f9df56e7 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -33,6 +33,11 @@ func TestParseMariaDBLogicalBackupFile(t *testing.T) { "backup.2026-10-21T10:00:00Z.gzip.sql": true, "backup.2026-10-21T10:00:00+04:00.sql": true, "backup.2026-10-21T10:00:00Z.sql.tmp": false, + "backup.2026-10-21T10:00:00Z.sql.gz": true, + "backup.2026-10-21T10:00:00Z.sql.zst": true, + "backup.2026-10-21T10:00:00Z.zstd.sql": true, + "backup.2026-10-21T10:00:00Z.xz.sql": false, + "backup.2026-10-21T10:00:00Z.sql.gz.tmp": false, "physicalbackup-20261021100000.xb": false, "backup.not-a-date.sql": false, "sub/backup.2026-10-21T10:00:00Z.sql": false, @@ -69,8 +74,8 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("single dump inside the backup window", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ - {Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111}, - {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242}, + {Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111, LastModified: created.Add(-24 * time.Hour)}, + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) @@ -96,7 +101,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}, } - l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3}}} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", tlsS3, mdbBackup, completed); got != 3 { t.Errorf("size = %d, want 3", got) @@ -114,7 +119,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "sts"}, Data: map[string][]byte{"token": []byte("TOKEN")}, } - l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4}}} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", stsS3, mdbBackup, completed); got != 4 { t.Errorf("size = %d, want 4", got) @@ -126,17 +131,30 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("pod clock skew within tolerance", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ - {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9}, + {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { t.Errorf("size = %d, want 9", got) } }) + t.Run("a skewed pod clock does not attribute a neighbouring run's dump", func(t *testing.T) { + // The pod clock runs 5m behind. This run's dump is named outside the + // window; the previous run's dump, uploaded before this run started, + // is named inside it. + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, + {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) + } + }) + t.Run("two dumps in the window leave size unset", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ - {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1}, - {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2}, + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, + {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) @@ -145,7 +163,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("no dump in the window leaves size unset", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ - {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1}, + {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) @@ -234,7 +252,7 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { }, } r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{ - {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048}, + {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)}, }}, job, s3CredsSecret("tenant", "creds")) rendered := newRenderedMariaDBTemplate() rendered.Storage.S3.Prefix = "src/" diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index bdffc7e7c..f6abed32d 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -421,7 +421,7 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { return nil, getErr } return existing, nil From 2b2176357761ac32a1215f19eeaf4e5cbde224a5 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 22 Oct 2026 18:18:51 +0400 Subject: [PATCH 11/20] fix(backups): do not fail a backup on a transient retry-check read The check for a Backup an earlier reconcile already created returned its read error, and the drivers turn any error from that path into a terminally Failed BackupJob. An apiserver timeout right after the operator backup completed therefore lost a backup whose data was safely in S3. The check only saves a probe, and the Create that follows already handles an existing Backup, so log the error and go on to Create. A trailing slash on the S3 endpoint is not a path prefix; the operators accept it, so do not refuse it. Pin the uncached AlreadyExists refetch, the upper slack of the MariaDB window and the probe timeout with tests. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 30 ++++--- .../backupcontroller/artifact_size_test.go | 85 +++++++++++++++++-- .../cnpg_artifact_size_test.go | 28 ++++++ .../cnpgstrategy_controller.go | 4 +- .../mariadb_artifact_size_test.go | 61 +++++++++++++ .../mariadbstrategy_controller.go | 4 +- 6 files changed, 190 insertions(+), 22 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 517fe07a6..182ea0b58 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -22,8 +22,8 @@ import ( // makes to measure Backup.status.artifact.sizeBytes. The probe runs inside // the reconcile that creates the Backup, and the BackupJob controller has a // single worker, so an unreachable endpoint stalls every other BackupJob for -// this long. -const artifactSizeProbeTimeout = 10 * time.Second +// this long. A variable so tests can shorten it. +var artifactSizeProbeTimeout = 10 * time.Second type s3ObjectInfo struct { Key string @@ -63,8 +63,10 @@ type minioObjectLister struct { func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { // An endpoint behind a path prefix cannot be addressed: minio-go has no // such option, and the SigV4 signature covers the request path, so the - // prefix cannot be added below the client either. - if strings.Contains(conn.Endpoint, "/") { + // prefix cannot be added below the client either. A bare trailing slash + // is not a prefix, and the operators accept it. + endpoint := strings.TrimSuffix(conn.Endpoint, "/") + if strings.Contains(endpoint, "/") { return nil, fmt.Errorf("endpoint %q has a path, which the S3 client cannot address", conn.Endpoint) } opts := &minio.Options{ @@ -90,7 +92,7 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { tr.TLSClientConfig.RootCAs = pool opts.Transport = tr } - client, err := minio.New(conn.Endpoint, opts) + client, err := minio.New(endpoint, opts) if err != nil { return nil, err } @@ -121,18 +123,20 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O // existingBackup returns the Backup a previous reconcile already created // for the job, so a retry skips the size probe whose result it would -// discard. Returns (nil, nil) when there is none. It reads past the cache: -// a retry typically follows a Create made moments earlier, which the cache -// may not show yet. -func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (*backupsv1alpha1.Backup, error) { +// discard. It reads past the cache: a retry typically follows a Create made +// moments earlier, which the cache may not show yet. It returns nil both +// when there is no Backup and when the read fails: it only saves a probe, +// and the Create that follows handles an existing Backup on its own, so a +// transient read error must not fail a backup that succeeded. +func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) *backupsv1alpha1.Backup { existing := &backupsv1alpha1.Backup{} if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { - if apierrors.IsNotFound(err) { - return nil, nil + if !apierrors.IsNotFound(err) { + getLogger(ctx).Info("cannot check for an existing Backup; creating it", "backup", name, "error", err) } - return nil, err + return nil } - return existing, nil + return existing } func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index f24079f8b..0e66968b5 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -5,12 +5,15 @@ import ( "context" "errors" "testing" + "time" "github.com/minio/minio-go/v7" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) @@ -87,8 +90,10 @@ func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) { if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org/prefix", Secure: true}); err == nil { t.Error("an endpoint with a path was accepted") } - if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org:9000", Secure: true}); err != nil { - t.Errorf("host:port endpoint rejected: %v", err) + for _, ok := range []string{"s3.example.org:9000", "s3.example.org/"} { + if _, err := newMinioObjectLister(s3Connection{Endpoint: ok, Secure: true}); err != nil { + t.Errorf("endpoint %q rejected: %v", ok, err) + } } } @@ -101,8 +106,78 @@ func TestExistingBackup_ReadsPastTheCache(t *testing.T) { Client: newMariaDBStrategyTestClient(t), APIReader: newMariaDBStrategyTestClient(t, backup), } - got, err := r.existingBackup(context.Background(), "tenant", "bj") - if err != nil || got == nil { - t.Fatalf("existingBackup = %v, %v; want the Backup the API server holds", got, err) + if got := r.existingBackup(context.Background(), "tenant", "bj"); got == nil { + t.Fatal("existingBackup = nil; want the Backup the API server holds") + } +} + +// staleCache wraps a fake client so that Get reports NotFound for Backups, +// the way an informer cache does right after another writer created one. +func staleCache(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) +} + +// notFoundOnce wraps a fake client whose first Backup Get reports NotFound: +// the retry check runs before the Backup exists, the refetch after Create +// collides with it. +func notFoundOnce(t *testing.T, c client.Client) client.Client { + t.Helper() + missed := false + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed { + missed = true + return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) +} + +func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) { + failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("etcdserver: request timed out") + }, + }) + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing} + if got := r.existingBackup(context.Background(), "tenant", "bj"); got != nil { + t.Errorf("existingBackup = %#v, want nil on a read error", got) + } +} + +// blockingLister answers nothing until the probe's context is done, the way +// a black-holed endpoint does. +type blockingLister struct{} + +func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +// withShortProbeTimeout shortens the probe timeout and fails the test if fn +// is not back well before the default timeout would have expired. +func withShortProbeTimeout(t *testing.T, fn func()) { + t.Helper() + saved := artifactSizeProbeTimeout + artifactSizeProbeTimeout = 50 * time.Millisecond + defer func() { artifactSizeProbeTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the probe did not give up at its timeout") } } diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 13b647bdd..70bb2a485 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -83,6 +83,16 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { } }) + t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) { + r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000"); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + }) + t.Run("an empty endpointURL means AWS", func(t *testing.T) { store := newSizedBarmanObjectStore() store.EndpointURL = "" @@ -306,3 +316,21 @@ func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { t.Errorf("S3 listed %d times for a Backup that already exists", l.calls) } } + +func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}} + store := newCNPGStrategyTestClient(t, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil || got == nil || got.Name != "bj-race" { + t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 5dabfa66c..712dd8713 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -815,8 +815,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { - if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil { - return existing, err + if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil { + return existing, nil } takenAt := metav1.Now() if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() { diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 6f9df56e7..91f555bbd 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -13,6 +13,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -151,6 +153,25 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } }) + t.Run("a dump uploaded just after Complete is inside the slack", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 5 { + t.Errorf("size = %d, want 5", got) + } + }) + + t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) { + r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + }) + t.Run("two dumps in the window leave size unset", func(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, @@ -298,3 +319,43 @@ func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact) } } + +func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, &fakeS3Lister{}, job) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("etcdserver: request timed out") + }, + }) + got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate()) + if err != nil || got == nil || got.Name != "bj-transient" { + t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil { + t.Errorf("the Backup was not created: %v", err) + } +} + +func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}} + store := newMariaDBStrategyTestClient(t, job, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store), Scheme: store.Scheme()} + got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{}) + if err != nil || got == nil || got.Name != "bj-race" { + t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index f6abed32d..8db41fb7f 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -365,8 +365,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( mdbBackup *mariadbtypes.Backup, rendered *strategyv1alpha1.MariaDBTemplate, ) (*backupsv1alpha1.Backup, error) { - if existing, err := r.existingBackup(ctx, j.Namespace, j.Name); err != nil || existing != nil { - return existing, err + if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil { + return existing, nil } takenAt := metav1.Now() if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() { From 523d74a5033cedb15158c5c3e3673c4e79d98fb7 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 22 Oct 2026 18:41:03 +0400 Subject: [PATCH 12/20] fix(backups): retry instead of failing when an existing Backup cannot be read When Create collides with a Backup an earlier reconcile already made and the read that follows fails, the drivers failed the BackupJob, although the backup and its Backup both exist. Return a distinct error instead, which the drivers hand back to the controller so the reconcile is retried with backoff and then finds the Backup. minio.New already refuses an endpoint with a path prefix and accepts a trailing slash, so drop the duplicate check in the shared client; the CNPG probe keeps its own check because it parses the URL itself and would otherwise drop the path before minio sees it. Say in the sizeBytes API doc what the value covers, such as the base backup without the shared WAL archive for CNPG, and that strategies which cannot attribute stored bytes to one backup leave it unset. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- api/backups/v1alpha1/backup_types.go | 11 ++-- internal/backupcontroller/artifact_size.go | 31 +++++++---- .../backupcontroller/artifact_size_test.go | 52 +++++++++++++++---- .../cnpg_artifact_size_test.go | 18 +++++++ .../cnpgstrategy_controller.go | 4 +- .../mariadb_artifact_size_test.go | 17 ++++++ .../mariadbstrategy_controller.go | 4 +- .../backups.cozystack.io_backups.yaml | 11 ++-- 8 files changed, 116 insertions(+), 32 deletions(-) diff --git a/api/backups/v1alpha1/backup_types.go b/api/backups/v1alpha1/backup_types.go index 483bb1fd8..541488202 100644 --- a/api/backups/v1alpha1/backup_types.go +++ b/api/backups/v1alpha1/backup_types.go @@ -37,9 +37,14 @@ type BackupArtifact struct { // For example: s3://bucket/prefix/file.tar.gz URI string `json:"uri"` - // SizeBytes is the size of the artifact in bytes, if known. Drivers - // leave it unset rather than writing 0 when they cannot measure the - // stored size, so an absent value means "unknown", not "empty". + // SizeBytes is the stored size in bytes of the files this backup owns, + // if known: the dump of a MariaDB backup, for example, or the base backup + // of a CNPG one, which leaves out the WAL archive the server's backups + // share and a point-in-time restore also reads. Strategies that cannot + // attribute stored bytes to a single backup, such as Velero's, whose + // repository deduplicates across backups, leave it unset. Drivers never + // write 0 for a size they cannot measure, so an absent value means + // "unknown", not "empty". // +optional SizeBytes int64 `json:"sizeBytes,omitempty"` diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 182ea0b58..5addefd5a 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "errors" "fmt" - "strings" "time" "github.com/minio/minio-go/v7" @@ -14,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) @@ -38,7 +38,7 @@ type s3ObjectLister interface { // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { - // Endpoint is host[:port], without a scheme or a path. + // Endpoint is host[:port], without a scheme. Endpoint string Secure bool Region string @@ -61,14 +61,6 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { - // An endpoint behind a path prefix cannot be addressed: minio-go has no - // such option, and the SigV4 signature covers the request path, so the - // prefix cannot be added below the client either. A bare trailing slash - // is not a prefix, and the operators accept it. - endpoint := strings.TrimSuffix(conn.Endpoint, "/") - if strings.Contains(endpoint, "/") { - return nil, fmt.Errorf("endpoint %q has a path, which the S3 client cannot address", conn.Endpoint) - } opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -92,7 +84,9 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { tr.TLSClientConfig.RootCAs = pool opts.Transport = tr } - client, err := minio.New(endpoint, opts) + // minio.New refuses an endpoint with a path prefix, which it cannot + // address, and accepts a bare trailing slash. + client, err := minio.New(conn.Endpoint, opts) if err != nil { return nil, err } @@ -121,6 +115,21 @@ func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3O return out, nil } +// errBackupNotReadBack reports that Create collided with an existing Backup +// that could then not be read. The backup itself succeeded, so callers +// retry the reconcile instead of failing the BackupJob. +var errBackupNotReadBack = errors.New("the Backup already exists but could not be read back") + +// failBackupArtifact handles an error from creating the Backup of a +// completed backup: errBackupNotReadBack is retried through the controller's +// backoff, anything else fails the BackupJob. +func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) { + if errors.Is(err, errBackupNotReadBack) { + return ctrl.Result{}, err + } + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) +} + // existingBackup returns the Backup a previous reconcile already created // for the job, so a retry skips the size probe whose result it would // discard. It reads past the cache: a retry typically follows a Create made diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 0e66968b5..69112cd9d 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "errors" + "fmt" "testing" "time" @@ -86,17 +87,6 @@ func TestCollectObjects(t *testing.T) { } } -func TestNewMinioObjectLister_RejectsEndpointPath(t *testing.T) { - if _, err := newMinioObjectLister(s3Connection{Endpoint: "s3.example.org/prefix", Secure: true}); err == nil { - t.Error("an endpoint with a path was accepted") - } - for _, ok := range []string{"s3.example.org:9000", "s3.example.org/"} { - if _, err := newMinioObjectLister(s3Connection{Endpoint: ok, Secure: true}); err != nil { - t.Errorf("endpoint %q rejected: %v", ok, err) - } - } -} - // TestExistingBackup_ReadsPastTheCache pins that the retry check sees a // Backup the cache does not show yet: the previous reconcile created it // moments earlier. @@ -181,3 +171,43 @@ func withShortProbeTimeout(t *testing.T, fn func()) { t.Fatal("the probe did not give up at its timeout") } } + +// failingReads wraps a fake client whose every Get fails, like an apiserver +// that times out. +func failingReads(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("etcdserver: request timed out") + }, + }) +} + +func TestFailBackupArtifact(t *testing.T) { + newJob := func() *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}} + } + + t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) { + j := newJob() + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)} + _, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack)) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want it returned so the reconcile is retried", err) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } + }) + + t.Run("any other error fails the BackupJob", func(t *testing.T) { + j := newJob() + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)} + if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil { + t.Fatalf("failBackupArtifact: %v", err) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed", j.Status.Phase) + } + }) +} diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 70bb2a485..f6ea74b6f 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -334,3 +334,21 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testi t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err) } } + +func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}} + store := newCNPGStrategyTestClient(t, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} + _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 712dd8713..ed994ed22 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -435,7 +435,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -877,7 +877,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( } existing := &backupsv1alpha1.Backup{} if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 91f555bbd..fd3ec30d4 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -359,3 +359,20 @@ func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *te t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err) } } + +func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}} + store := newMariaDBStrategyTestClient(t, job, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store), Scheme: store.Scheme()} + _, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{}) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 8db41fb7f..4c920dcaa 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -214,7 +214,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 } artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -422,7 +422,7 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( } existing := &backupsv1alpha1.Backup{} if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index f30d3dbc7..dfa25dbc5 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -132,9 +132,14 @@ spec: type: string sizeBytes: description: |- - SizeBytes is the size of the artifact in bytes, if known. Drivers - leave it unset rather than writing 0 when they cannot measure the - stored size, so an absent value means "unknown", not "empty". + SizeBytes is the stored size in bytes of the files this backup owns, + if known: the dump of a MariaDB backup, for example, or the base backup + of a CNPG one, which leaves out the WAL archive the server's backups + share and a point-in-time restore also reads. Strategies that cannot + attribute stored bytes to a single backup, such as Velero's, whose + repository deduplicates across backups, leave it unset. Drivers never + write 0 for a size they cannot measure, so an absent value means + "unknown", not "empty". format: int64 type: integer uri: From 080df1b2f204377b24da31ad399df21b63179a07 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 22 Oct 2026 19:38:00 +0400 Subject: [PATCH 13/20] fix(backups): surface a Backup that cannot be read back and skip its probe A Backup that exists but cannot be read back is retried through the controller's backoff, which left the BackupJob Running with nothing on it saying why. Set Ready=False with reason BackupNotReadBack before handing the error back. When the check for an existing Backup itself fails, the Backup may well exist, so a size probe would only be discarded, and a read that keeps failing reran it, up to the probe timeout on the single BackupJob worker, on every retry. Skip the probe in that case and create the Backup without a size. Drive both behaviours through the MariaDB and CNPG reconciles in tests. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 36 ++++++---- .../backupcontroller/artifact_size_test.go | 20 ++++-- .../cnpg_artifact_size_test.go | 65 +++++++++++++++++++ .../cnpgstrategy_controller.go | 9 ++- .../mariadb_artifact_size_test.go | 65 +++++++++++++++++-- .../mariadbstrategy_controller.go | 9 +-- 6 files changed, 175 insertions(+), 29 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 5addefd5a..534019184 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -12,6 +12,8 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -122,9 +124,19 @@ var errBackupNotReadBack = errors.New("the Backup already exists but could not b // failBackupArtifact handles an error from creating the Backup of a // completed backup: errBackupNotReadBack is retried through the controller's -// backoff, anything else fails the BackupJob. +// backoff, with Ready=False telling why the BackupJob is still Running; +// anything else fails the BackupJob. func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) { if errors.Is(err, errBackupNotReadBack) { + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "BackupNotReadBack", + Message: err.Error(), + }) + if uerr := r.Status().Update(ctx, j); uerr != nil { + getLogger(ctx).Info("cannot record that the Backup could not be read back", "backupjob", j.Name, "error", uerr) + } return ctrl.Result{}, err } return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) @@ -133,19 +145,21 @@ func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backups // existingBackup returns the Backup a previous reconcile already created // for the job, so a retry skips the size probe whose result it would // discard. It reads past the cache: a retry typically follows a Create made -// moments earlier, which the cache may not show yet. It returns nil both -// when there is no Backup and when the read fails: it only saves a probe, -// and the Create that follows handles an existing Backup on its own, so a -// transient read error must not fail a backup that succeeded. -func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) *backupsv1alpha1.Backup { - existing := &backupsv1alpha1.Backup{} +// moments earlier, which the cache may not show yet. A failed read is not +// fatal, because the Create that follows handles an existing Backup on its +// own; probe is false then, since the Backup may well exist and a probe +// would only be discarded, while a read that keeps failing would rerun it on +// every retry. +func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (existing *backupsv1alpha1.Backup, probe bool) { + existing = &backupsv1alpha1.Backup{} if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { - if !apierrors.IsNotFound(err) { - getLogger(ctx).Info("cannot check for an existing Backup; creating it", "backup", name, "error", err) + if apierrors.IsNotFound(err) { + return nil, true } - return nil + getLogger(ctx).Info("cannot check for an existing Backup; creating it without measuring its size", "backup", name, "error", err) + return nil, false } - return existing + return existing, false } func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 69112cd9d..523d56469 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -12,6 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -96,9 +97,13 @@ func TestExistingBackup_ReadsPastTheCache(t *testing.T) { Client: newMariaDBStrategyTestClient(t), APIReader: newMariaDBStrategyTestClient(t, backup), } - if got := r.existingBackup(context.Background(), "tenant", "bj"); got == nil { + if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil { t.Fatal("existingBackup = nil; want the Backup the API server holds") } + r.APIReader = newMariaDBStrategyTestClient(t) + if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe { + t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe) + } } // staleCache wraps a fake client so that Get reports NotFound for Backups, @@ -139,8 +144,8 @@ func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) { }, }) r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing} - if got := r.existingBackup(context.Background(), "tenant", "bj"); got != nil { - t.Errorf("existingBackup = %#v, want nil on a read error", got) + if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe { + t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe) } } @@ -195,9 +200,16 @@ func TestFailBackupArtifact(t *testing.T) { if !errors.Is(err, errBackupNotReadBack) { t.Errorf("err = %v, want it returned so the reconcile is retried", err) } - if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + persisted := &backupsv1alpha1.BackupJob{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { t.Error("the BackupJob was failed although its Backup exists") } + if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" { + t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond) + } }) t.Run("any other error fails the BackupJob", func(t *testing.T) { diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index f6ea74b6f..7b3852317 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -10,9 +10,11 @@ import ( "net/http/httptest" "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -352,3 +354,66 @@ func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T t.Errorf("err = %v, want errBackupNotReadBack", err) } } + +// TestReconcileCNPG_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileCNPG_UnreadableBackupIsRetried(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + ctx := context.Background() + completed := backups[0].DeepCopy() + completed.Status.Phase = cnpgBackupPhaseComplete + completed.Status.BackupID = "20261021T100000" + if err := f.c.Update(ctx, completed); err != nil { + t.Fatalf("complete cnpg.io/Backup: %v", err) + } + if err := f.c.Create(ctx, &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "bj"}}); err != nil { + t.Fatalf("seed Backup: %v", err) + } + f.r.APIReader = failingReads(t, f.c) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileCNPG err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} + +func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + r.APIReader = failingReads(t, newCNPGStrategyTestClient(t)) + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil || got == nil { + t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index ed994ed22..819efc10f 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -815,7 +815,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { - if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { return existing, nil } takenAt := metav1.Now() @@ -858,12 +859,14 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( Status: backupsv1alpha1.BackupStatus{ Phase: backupsv1alpha1.BackupPhaseReady, Artifact: &backupsv1alpha1.BackupArtifact{ - URI: fmt.Sprintf("cnpg://%s/%s", serverName, cnpgBackup.Name), - SizeBytes: r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID), + URI: fmt.Sprintf("cnpg://%s/%s", serverName, cnpgBackup.Name), }, UnderlyingResources: underlyingResources, }, } + if probe { + backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID) + } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef } diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index fd3ec30d4..b49bcae66 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -13,8 +13,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -328,12 +328,9 @@ func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, }, } - r := reconcilerWithLister(t, &fakeS3Lister{}, job) - r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ - Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return errors.New("etcdserver: request timed out") - }, - }) + l := &fakeS3Lister{} + r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds")) + r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t)) got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate()) if err != nil || got == nil || got.Name != "bj-transient" { t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) @@ -341,6 +338,9 @@ func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil { t.Errorf("the Backup was not created: %v", err) } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } } func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) { @@ -376,3 +376,54 @@ func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testin t.Errorf("err = %v, want errBackupNotReadBack", err) } } + +// TestReconcileMariaDB_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) { + strategyGroup := strategyv1alpha1.GroupVersion.Group + now := metav1.Now() + bj := newMariaDBBackupJob("bj-unreadable", "tenant") + bj.Status.StartedAt = &now + bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}} + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: bj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: now, + }}}, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: bj.Name}} + store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup, backup) + r := &BackupJobReconciler{ + Client: store, + APIReader: failingReads(t, store), + Scheme: store.Scheme(), + Recorder: record.NewFakeRecorder(10), + } + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name, + }} + + _, err := r.reconcileMariaDB(context.Background(), bj, resolved) + if !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileMariaDB err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 4c920dcaa..749d63c9c 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -365,7 +365,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( mdbBackup *mariadbtypes.Backup, rendered *strategyv1alpha1.MariaDBTemplate, ) (*backupsv1alpha1.Backup, error) { - if existing := r.existingBackup(ctx, j.Namespace, j.Name); existing != nil { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { return existing, nil } takenAt := metav1.Now() @@ -395,9 +396,9 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( // operator's PVC ID), so an Artifact{URI: ""} is a worse signal than // no Artifact at all - leave it nil instead. if uri := driverMD[mariadbStorageURIKey]; uri != "" { - status.Artifact = &backupsv1alpha1.BackupArtifact{ - URI: uri, - SizeBytes: r.mariadbArtifactSizeBytes(ctx, j.Namespace, rendered.Storage.S3, mdbBackup, takenAt.Time), + status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} + if probe { + status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, j.Namespace, rendered.Storage.S3, mdbBackup, takenAt.Time) } } backup := &backupsv1alpha1.Backup{ From 92fa97969016ca92a4b123bcd4c11c710f6cfd32 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Thu, 22 Oct 2026 20:08:29 +0400 Subject: [PATCH 14/20] fix(backups): trust only the endpoint CA where the backup writer does With an endpoint CA the probe trusted the system roots plus that CA. barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the default trust store, so a CNPG probe could accept an endpoint the backup itself would have refused. Let the connection say whether the CA adds to the system roots or replaces them, and replace them for CNPG. mariadb-operator appends its CA to the system pool, so MariaDB keeps that. Pin in reconcile tests that a Create the apiserver rejects still fails the BackupJob, so only the read-back of an existing Backup is retried. Assisted-by: LLM Signed-off-by: Andrey Kolkov --- internal/backupcontroller/artifact_size.go | 38 ++++++++---- .../backupcontroller/artifact_size_test.go | 62 +++++++++++++++++++ .../backupcontroller/cnpg_artifact_size.go | 4 +- .../cnpg_artifact_size_test.go | 36 ++++++++++- .../mariadb_artifact_size_test.go | 48 ++++++++++++++ 5 files changed, 175 insertions(+), 13 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 534019184..1a822699c 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -47,8 +47,11 @@ type s3Connection struct { AccessKey string SecretKey string SessionToken string - // CAPEM is trusted on top of the system pool when set. - CAPEM []byte + // CAPEM is an endpoint CA bundle. It is trusted on top of the system + // roots, or instead of them when CAOnly is set: the probe must trust + // what the writer of the backup trusted, no more. + CAPEM []byte + CAOnly bool } func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { @@ -74,16 +77,9 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { if err != nil { return nil, err } - pool := tr.TLSClientConfig.RootCAs - if pool == nil { - if pool, err = x509.SystemCertPool(); err != nil { - pool = x509.NewCertPool() - } + if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { + return nil, err } - if !pool.AppendCertsFromPEM(conn.CAPEM) { - return nil, errors.New("no PEM certificates in the endpoint CA bundle") - } - tr.TLSClientConfig.RootCAs = pool opts.Transport = tr } // minio.New refuses an endpoint with a path prefix, which it cannot @@ -95,6 +91,26 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { return &minioObjectLister{client: client}, nil } +// endpointTrustPool returns the roots to verify the endpoint against: the +// CA bundle on top of system, or the CA bundle alone with CAOnly. A nil +// system means the platform's roots. +func endpointTrustPool(conn s3Connection, system *x509.CertPool) (*x509.CertPool, error) { + pool := system + switch { + case conn.CAOnly: + pool = x509.NewCertPool() + case pool == nil: + var err error + if pool, err = x509.SystemCertPool(); err != nil { + pool = x509.NewCertPool() + } + } + if !pool.AppendCertsFromPEM(conn.CAPEM) { + return nil, errors.New("no PEM certificates in the endpoint CA bundle") + } + return pool, nil +} + func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 523d56469..3d9c35b23 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -3,8 +3,17 @@ package backupcontroller import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" + "math/big" + "net/http" + "net/http/httptest" "testing" "time" @@ -223,3 +232,56 @@ func TestFailBackupArtifact(t *testing.T) { } }) } + +// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system +// roots" and "the CA alone". The system pool holds the endpoint's +// certificate, and the configured CA is an unrelated one, so the endpoint +// must stop being trusted only when CAOnly is set. +func TestEndpointTrustPool_CAOnly(t *testing.T) { + endpoint := httptest.NewTLSServer(http.NotFoundHandler()) + defer endpoint.Close() + system := x509.NewCertPool() + system.AddCert(endpoint.Certificate()) + + // Every httptest server shares one certificate, so the unrelated CA is + // minted here. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + + verifies := func(caOnly bool) bool { + pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone()) + if err != nil { + t.Fatalf("endpointTrustPool: %v", err) + } + _, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool}) + return err == nil + } + if !verifies(false) { + t.Error("CA on top of the system roots: the endpoint was not trusted") + } + if verifies(true) { + t.Error("CA alone: the endpoint was trusted through the system roots") + } +} + +// forbidBackupCreate wraps a fake client whose Create of a Backup is +// rejected, the way an admission webhook rejects one. +func forbidBackupCreate(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy")) + } + return cl.Create(ctx, obj, opts...) + }, + }) +} diff --git a/internal/backupcontroller/cnpg_artifact_size.go b/internal/backupcontroller/cnpg_artifact_size.go index c7d91bdb2..0756e2bb9 100644 --- a/internal/backupcontroller/cnpg_artifact_size.go +++ b/internal/backupcontroller/cnpg_artifact_size.go @@ -105,7 +105,9 @@ func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace st if err != nil { return conn, err } - conn.CAPEM = []byte(pem) + // barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the + // default trust store rather than adding to it. + conn.CAPEM, conn.CAOnly = []byte(pem), true } return conn, nil } diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 7b3852317..85e55e8c2 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -126,7 +126,7 @@ func TestCNPGArtifactSizeBytes(t *testing.T) { if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 { t.Errorf("size = %d, want 5", got) } - if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" { + if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" || !l.conn.CAOnly { t.Errorf("connection = %+v", l.conn) } }) @@ -417,3 +417,37 @@ func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) { t.Errorf("S3 probed %d times after the existence check failed", l.calls) } } + +// TestReconcileCNPG_RejectedCreateFailsTheJob pins that only the read-back +// is retried: a Create the apiserver rejects still fails the BackupJob. +func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + ctx := context.Background() + completed := backups[0].DeepCopy() + completed.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(ctx, completed); err != nil { + t.Fatalf("complete cnpg.io/Backup: %v", err) + } + f.r.Client = forbidBackupCreate(t, f.c) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil { + t.Fatalf("reconcileCNPG err = %v, want the BackupJob failed rather than retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) + } +} diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index b49bcae66..084abe31c 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -427,3 +427,51 @@ func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) { t.Error("the BackupJob was failed although its Backup exists") } } + +// TestReconcileMariaDB_RejectedCreateFailsTheJob pins that only the +// read-back is retried: a Create the apiserver rejects still fails the +// BackupJob. +func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) { + strategyGroup := strategyv1alpha1.GroupVersion.Group + now := metav1.Now() + bj := newMariaDBBackupJob("bj-rejected", "tenant") + bj.Status.StartedAt = &now + bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}} + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: bj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: now, + }}}, + } + store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup) + r := &BackupJobReconciler{ + Client: forbidBackupCreate(t, store), + Scheme: store.Scheme(), + Recorder: record.NewFakeRecorder(10), + } + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name, + }} + + if _, err := r.reconcileMariaDB(context.Background(), bj, resolved); err != nil { + t.Fatalf("reconcileMariaDB err = %v, want the BackupJob failed rather than retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) + } +} From c140354977cda456d63c55d8cf3f12c0f6b6f33c Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 13:13:54 +0000 Subject: [PATCH 15/20] test(backups): build operator Backups with the operator's success shape mariadb-operator marks a Backup Complete=True for a failed Job as well as a successful one; only the reason tells them apart (JobComplete with message Success, or JobFailed with message Failed). The fixtures that stand for a finished backup set Complete=True and nothing else, so they describe a state the operator never produces and would read as a failure to a driver that checks the reason. Give them the success shape the operator writes. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../mariadb_artifact_size_test.go | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 084abe31c..d4bf91a1d 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -29,6 +29,19 @@ func dumpKey(prefix string, ts time.Time, compression string) string { return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) } +// mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a +// Backup whose Job succeeded. A failed Job also gets Status=True, with +// Reason=JobFailed, so a fixture without the reason is not a success. +func mariadbBackupSucceeded(at metav1.Time) metav1.Condition { + return metav1.Condition{ + Type: mariadbtypes.ConditionTypeComplete, + Status: metav1.ConditionTrue, + Reason: mariadbtypes.ConditionReasonJobComplete, + Message: "Success", + LastTransitionTime: at, + } +} + func TestParseMariaDBLogicalBackupFile(t *testing.T) { cases := map[string]bool{ "backup.2026-10-21T10:00:00Z.sql": true, @@ -258,9 +271,7 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { Namespace: "tenant", Name: "src-op-bk", CreationTimestamp: metav1.NewTime(completed.Add(-3 * time.Minute)), }, - Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ - Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: completed, - }}}, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(completed)}}, } resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, @@ -399,9 +410,7 @@ func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) { backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, }, }, - Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ - Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: now, - }}}, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}}, } backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: bj.Name}} store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup, backup) @@ -450,9 +459,7 @@ func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) { backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, }, }, - Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{ - Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: now, - }}}, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}}, } store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup) r := &BackupJobReconciler{ From 21f51a2e74c3cb3cabb639d192f7953e0028837f Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 13:16:15 +0000 Subject: [PATCH 16/20] refactor(backups): return the matched MariaDB dump, not only its size A MariaDB restore has to name the dump it restores from: without a target time the operator restores the newest dump under the prefix, which is not necessarily the one of the Backup being restored. The size probe already finds that dump, but it returns only its size, folds a missing, an ambiguous and an unreadable listing into the same 0, and lives on BackupJobReconciler, which the restore reconciler is not. Split the probe in two: findMariaDBDump reads the bucket and returns the matched object (key, file name, the timestamp in its name, size and LastModified) or an error that tells "not found" from "ambiguous"; the size path keeps its timeout and reads only the size. The Secret lookup and the S3 client move onto an objectStore value that any reconciler can build from its own client, and BackupJobReconciler keeps one-line wrappers so the CNPG probe is unchanged. sizeBytes is reported exactly as before. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- internal/backupcontroller/artifact_size.go | 48 ++++++-- .../backupcontroller/mariadb_artifact_size.go | 104 ++++++++++++------ .../mariadb_artifact_size_test.go | 32 ++++++ 3 files changed, 140 insertions(+), 44 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 1a822699c..063e165d6 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) @@ -54,13 +55,44 @@ type s3Connection struct { CAOnly bool } -func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { - if r.newS3ObjectLister != nil { - return r.newS3ObjectLister(conn) +// objectStore reads back what a backup wrote: the Secrets it was written +// with and the bucket itself. It is a value rather than reconciler methods +// so that the restore path, which has a reconciler of its own, can read +// the same objects. +type objectStore struct { + // reader must reach Secrets; the manager's client reads them uncached. + reader client.Reader + // newLister overrides the S3 client; nil means a real minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) +} + +func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { + if s.newLister != nil { + return s.newLister(conn) } return newMinioObjectLister(conn) } +func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { + secret := &corev1.Secret{} + if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", err + } + v, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key) + } + return string(v), nil +} + +func (r *BackupJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + +func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { + return r.objectStore().lister(conn) +} + type minioObjectLister struct { client *minio.Client } @@ -179,13 +211,5 @@ func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, nam } func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { - secret := &corev1.Secret{} - if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { - return "", err - } - v, ok := secret.Data[key] - if !ok { - return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key) - } - return string(v), nil + return r.objectStore().secretValue(ctx, namespace, name, key) } diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 273eb8105..bf9dedcd9 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -3,6 +3,8 @@ package backupcontroller import ( "context" + "errors" + "fmt" "strings" "time" @@ -16,15 +18,28 @@ import ( // the object store's clock, neither of which is the apiserver's. const mariadbSizeProbeClockSkew = time.Minute +var ( + errMariaDBDumpNotFound = errors.New("no dump in the bucket belongs to the operator Backup") + errMariaDBDumpAmbiguous = errors.New("the dump of the operator Backup cannot be told apart from another run's") +) + +// mariadbDump is a logical dump mariadb-operator wrote to the bucket. +type mariadbDump struct { + // Key is the object key; Name is the file name, Key without the prefix. + Key string + Name string + // Timestamp is the one in the file name, taken by the backup Job pod's + // clock as the dump started. + Timestamp time.Time + Size int64 + // LastModified is the object store's clock as the upload finished. + LastModified time.Time +} + // mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for -// mdbBackup. The strategy prefix is shared by every backup of the -// application, so the dump is identified by both the timestamp in its file -// name and its LastModified falling inside the operator Backup's lifetime: -// the two come from different clocks, so a skewed pod clock alone cannot -// pull a neighbouring run's dump into the window. Returns 0 - leaving -// Artifact.SizeBytes absent - when the object store cannot be read or the -// window does not match exactly one dump: an unknown size must not be -// reported as a measured one. +// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump +// cannot be found or told apart: an unknown size must not be reported as a +// measured one. func (r *BackupJobReconciler) mariadbArtifactSizeBytes( ctx context.Context, namespace string, @@ -32,19 +47,33 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( mdbBackup *mariadbtypes.Backup, completedAt time.Time, ) int64 { - logger := getLogger(ctx) ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() - - conn, err := r.mariadbS3Connection(ctx, namespace, s3) + dump, err := findMariaDBDump(ctx, r.objectStore(), namespace, s3, mdbBackup, completedAt) if err != nil { - logger.Info("cannot resolve S3 credentials to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) + getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) return 0 } - lister, err := r.s3Lister(conn) + return dump.Size +} + +// findMariaDBDump lists the bucket the operator Backup wrote to and returns +// the one dump that belongs to it, as matchMariaDBDump decides. +func findMariaDBDump( + ctx context.Context, + store objectStore, + namespace string, + s3 *strategyv1alpha1.MariaDBS3Template, + mdbBackup *mariadbtypes.Backup, + completedAt time.Time, +) (mariadbDump, error) { + conn, err := mariadbS3Connection(ctx, store, namespace, s3) if err != nil { - logger.Info("cannot build S3 client to measure MariaDB backup size", "backup", mdbBackup.Name, "error", err) - return 0 + return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same @@ -52,51 +81,62 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( prefix := mariadbOperatorS3Prefix(s3.Prefix) objects, err := lister.ListObjects(ctx, s3.Bucket, prefix, false) if err != nil { - logger.Info("cannot list MariaDB backup objects to measure size", "backup", mdbBackup.Name, "error", err) - return 0 + return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err) } + return matchMariaDBDump(objects, prefix, mdbBackup.CreationTimestamp.Time, completedAt) +} - lower := mdbBackup.CreationTimestamp.Add(-mariadbSizeProbeClockSkew) - upper := completedAt.Add(mariadbSizeProbeClockSkew) - var matched []s3ObjectInfo +// matchMariaDBDump picks the dump of the run that lived from created to +// completed. The prefix is shared by every backup of the application, so +// the dump is identified by both the timestamp in its file name and its +// LastModified falling inside that lifetime: the two come from different +// clocks, so a skewed pod clock alone cannot pull a neighbouring run's dump +// into the window. +func matchMariaDBDump(objects []s3ObjectInfo, prefix string, created, completed time.Time) (mariadbDump, error) { + lower := created.Add(-mariadbSizeProbeClockSkew) + upper := completed.Add(mariadbSizeProbeClockSkew) + var matched []mariadbDump for _, obj := range objects { - ts, ok := parseMariaDBLogicalBackupFile(strings.TrimPrefix(obj.Key, prefix)) + name := strings.TrimPrefix(obj.Key, prefix) + ts, ok := parseMariaDBLogicalBackupFile(name) if !ok || !inWindow(ts, lower, upper) || !inWindow(obj.LastModified, lower, upper) { continue } - matched = append(matched, obj) + matched = append(matched, mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified}) } - if len(matched) != 1 { - logger.Info("MariaDB backup dump not attributable to a single object; leaving size unset", - "backup", mdbBackup.Name, "candidates", len(matched)) - return 0 + switch len(matched) { + case 0: + return mariadbDump{}, errMariaDBDumpNotFound + case 1: + return matched[0], nil + default: + return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime", errMariaDBDumpAmbiguous, len(matched)) } - return matched[0].Size } // mariadbS3Connection resolves the Secrets and TLS settings the operator // Backup was rendered with, so the bucket is read with the tenant's // credentials rather than the controller's. -func (r *BackupJobReconciler) mariadbS3Connection(ctx context.Context, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3Connection, error) { +func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3Connection, error) { conn := s3Connection{ Endpoint: s3.Endpoint, Region: s3.Region, Secure: s3.TLS != nil && s3.TLS.Enabled, } var err error - if conn.AccessKey, err = r.secretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef.Name, s3.AccessKeyIDSecretKeyRef.Key); err != nil { + if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef.Name, s3.AccessKeyIDSecretKeyRef.Key); err != nil { return conn, err } - if conn.SecretKey, err = r.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil { + if conn.SecretKey, err = store.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil { return conn, err } if ref := s3.SessionTokenSecretKeyRef; ref != nil { - if conn.SessionToken, err = r.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil { + if conn.SessionToken, err = store.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil { return conn, err } } if conn.Secure && s3.TLS.CASecretKeyRef != nil { - ca, err := r.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) + ca, err := store.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) if err != nil { return conn, err } diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index d4bf91a1d..c28d0ad4e 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -220,6 +220,38 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { }) } +// TestMatchMariaDBDump pins what a restore reads off a match: the object +// itself, and errors it can tell apart. +func TestMatchMariaDBDump(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + const prefix = "tenant/db/" + own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)} + + got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, created, completed) + if err != nil { + t.Fatalf("matchMariaDBDump: %v", err) + } + want := mariadbDump{ + Key: own.Key, + Name: strings.TrimPrefix(own.Key, prefix), + Timestamp: created.Add(time.Minute), + Size: 4242, + LastModified: own.LastModified, + } + if got != want { + t.Errorf("matchMariaDBDump = %+v, want %+v", got, want) + } + + if _, err := matchMariaDBDump(nil, prefix, created, completed); !errors.Is(err, errMariaDBDumpNotFound) { + t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err) + } + second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)} + if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, created, completed); !errors.Is(err, errMariaDBDumpAmbiguous) { + t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err) + } +} + // TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client // against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and // response parsing are exercised together. From 560c44f71eba87c6bb08bb119e7572abdc9bfc69 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 13:18:39 +0000 Subject: [PATCH 17/20] fix(backups): read a MariaDB dump from where its operator Backup wrote it The size probe took the bucket, endpoint, prefix, credentials and TLS settings from the strategy as it renders on the reconcile that creates the Backup, which is not necessarily what it rendered when the operator Backup was created: an existing operator Backup is reused untouched, and a backup can run for up to the driver's 30-minute deadline. An admin who edits the strategy in between sends the probe to the new location, where it measures the wrong prefix or nothing at all. The operator Backup's own spec.storage.s3 is where the dump went, so the probe reads that. It is also the only source a restore can use, since it targets that operator Backup long after the strategy may have changed. An operator Backup that does not store to S3 is refused before anything is listed. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../backupcontroller/mariadb_artifact_size.go | 43 +++--- .../mariadb_artifact_size_test.go | 139 ++++++++++++++---- .../mariadbstrategy_controller.go | 2 +- 3 files changed, 131 insertions(+), 53 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index bf9dedcd9..d10f1c4a5 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -8,7 +8,6 @@ import ( "strings" "time" - strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) @@ -40,16 +39,10 @@ type mariadbDump struct { // mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump // cannot be found or told apart: an unknown size must not be reported as a // measured one. -func (r *BackupJobReconciler) mariadbArtifactSizeBytes( - ctx context.Context, - namespace string, - s3 *strategyv1alpha1.MariaDBS3Template, - mdbBackup *mariadbtypes.Backup, - completedAt time.Time, -) int64 { +func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() - dump, err := findMariaDBDump(ctx, r.objectStore(), namespace, s3, mdbBackup, completedAt) + dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) if err != nil { getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) return 0 @@ -57,17 +50,17 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes( return dump.Size } -// findMariaDBDump lists the bucket the operator Backup wrote to and returns -// the one dump that belongs to it, as matchMariaDBDump decides. -func findMariaDBDump( - ctx context.Context, - store objectStore, - namespace string, - s3 *strategyv1alpha1.MariaDBS3Template, - mdbBackup *mariadbtypes.Backup, - completedAt time.Time, -) (mariadbDump, error) { - conn, err := mariadbS3Connection(ctx, store, namespace, s3) +// findMariaDBDump lists the bucket mdbBackup wrote to and returns the one +// dump that belongs to it, as matchMariaDBDump decides. The location and +// credentials come from mdbBackup's own spec: the strategy it was rendered +// from can have been edited since, and pointing at another bucket now says +// nothing about where this dump went. +func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) if err != nil { return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) } @@ -114,17 +107,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, created, completed } } -// mariadbS3Connection resolves the Secrets and TLS settings the operator -// Backup was rendered with, so the bucket is read with the tenant's -// credentials rather than the controller's. -func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *strategyv1alpha1.MariaDBS3Template) (s3Connection, error) { +// mariadbS3Connection resolves the Secrets and TLS settings of the operator +// Backup's storage, so the bucket is read with the tenant's credentials +// rather than the controller's. +func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *mariadbtypes.S3Storage) (s3Connection, error) { conn := s3Connection{ Endpoint: s3.Endpoint, Region: s3.Region, Secure: s3.TLS != nil && s3.TLS.Enabled, } var err error - if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIDSecretKeyRef.Name, s3.AccessKeyIDSecretKeyRef.Key); err != nil { + if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIdSecretKeyRef.Name, s3.AccessKeyIdSecretKeyRef.Key); err != nil { return conn, err } if conn.SecretKey, err = store.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil { diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index c28d0ad4e..719678892 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -77,14 +77,26 @@ func TestMariaDBOperatorS3Prefix(t *testing.T) { } } +// operatorBackupStoringTo returns the operator Backup a strategy with s3 +// storage renders, created at created in namespace tenant. +func operatorBackupStoringTo(t *testing.T, s3 *strategyv1alpha1.MariaDBS3Template, created time.Time) *mariadbtypes.Backup { + t.Helper() + storage, err := buildMariaDBBackupStorage(strategyv1alpha1.MariaDBStorageTemplate{S3: s3}) + if err != nil { + t.Fatal(err) + } + return &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created)}, + Spec: mariadbtypes.BackupSpec{Storage: storage}, + } +} + func TestMariaDBArtifactSizeBytes(t *testing.T) { created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) completed := created.Add(3 * time.Minute) - mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ - Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), - }} s3 := newRenderedMariaDBTemplate().Storage.S3 s3.Prefix = "tenant/db" + mdbBackup := operatorBackupStoringTo(t, s3, created) const prefix = "tenant/db/" t.Run("single dump inside the backup window", func(t *testing.T) { @@ -93,7 +105,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed) + got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) if got != 4242 { t.Errorf("size = %d, want 4242", got) } @@ -118,7 +130,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) - if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", tlsS3, mdbBackup, completed); got != 3 { + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { t.Errorf("size = %d, want 3", got) } if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { @@ -136,7 +148,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) - if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", stsS3, mdbBackup, completed); got != 4 { + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { t.Errorf("size = %d, want 4", got) } if l.conn.SessionToken != "TOKEN" { @@ -148,7 +160,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 9 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -161,7 +173,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) } }) @@ -170,7 +182,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 5 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { t.Errorf("size = %d, want 5", got) } }) @@ -179,7 +191,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } withShortProbeTimeout(t, func() { - if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -190,7 +202,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -199,14 +211,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -214,7 +226,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} - if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, completed); got != 0 { + if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -252,6 +264,45 @@ func TestMatchMariaDBDump(t *testing.T) { } } +// TestFindMariaDBDump pins that a dump is read with nothing but a client, +// as a reconciler other than BackupJobReconciler has, and that an operator +// Backup that does not store to S3 is refused without a listing. +func TestFindMariaDBDump(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("tenant/db/", created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, + }} + store := objectStore{ + reader: newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")), + newLister: func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + }, + } + + got, err := findMariaDBDump(context.Background(), store, operatorBackupStoringTo(t, s3, created), completed) + if err != nil || got.Size != 4242 { + t.Fatalf("findMariaDBDump = %+v, %v; want the 4242-byte dump", got, err) + } + if l.conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the Secret's key", l.conn) + } + + pvc := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-pvc", CreationTimestamp: metav1.NewTime(created)}, + Spec: mariadbtypes.BackupSpec{Storage: mariadbtypes.BackupStorage{PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{}}}, + } + if _, err := findMariaDBDump(context.Background(), store, pvc, completed); err == nil { + t.Error("findMariaDBDump on a PVC-backed operator Backup returned no error") + } + if l.calls != 1 { + t.Errorf("S3 listed %d times, want once: a PVC-backed Backup has nothing to list", l.calls) + } +} + // TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client // against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and // response parsing are exercised together. @@ -280,11 +331,8 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { s3.Endpoint = strings.TrimPrefix(srv.URL, "http://") s3.Prefix = "tenant/db/" s3.Region = "us-east-1" - mdbBackup := &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{ - Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created), - }} - if got := r.mariadbArtifactSizeBytes(context.Background(), "tenant", s3, mdbBackup, created.Add(3*time.Minute)); got != 31337 { + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { t.Fatalf("size = %d, want 31337", got) } if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { @@ -298,13 +346,11 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { apps := mariadbapp.GroupName completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) - mdbBackup := &mariadbtypes.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "tenant", Name: "src-op-bk", - CreationTimestamp: metav1.NewTime(completed.Add(-3 * time.Minute)), - }, - Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(completed)}}, - } + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, completed.Add(-3*time.Minute)) + mdbBackup.Name = "src-op-bk" + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} resolved := &ResolvedBackupConfig{ StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, Parameters: map[string]string{}, @@ -318,8 +364,6 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)}, }}, job, s3CredsSecret("tenant", "creds")) - rendered := newRenderedMariaDBTemplate() - rendered.Storage.S3.Prefix = "src/" artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered) if err != nil { @@ -330,6 +374,47 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { } } +// TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the +// probe reads where the operator Backup wrote its dump, not where the +// strategy, edited while the backup ran, now points. +func TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup(t *testing.T) { + apps := mariadbapp.GroupName + completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) + original := newRenderedMariaDBTemplate().Storage.S3 + original.Prefix = "original/" + mdbBackup := operatorBackupStoringTo(t, original, completed.Add(-3*time.Minute)) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + edited := newRenderedMariaDBTemplate() + edited.Storage.S3.Bucket = "replacement-bkt" + edited.Storage.S3.Endpoint = "replacement.example:9000" + edited.Storage.S3.Prefix = "replacement/" + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-edited"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("original/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)}, + }} + r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds")) + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + + artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, edited) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if l.bucket != "bkt" || l.prefix != "original/" || l.conn.Endpoint != "s3.example:9000" { + t.Errorf("listed %s/%s at %s, want bkt/original/ at s3.example:9000, where the operator Backup wrote", l.bucket, l.prefix, l.conn.Endpoint) + } + if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 { + t.Errorf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) + } +} + func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { apps := mariadbapp.GroupName job := &backupsv1alpha1.BackupJob{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 749d63c9c..4d88ab8a1 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -398,7 +398,7 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} if probe { - status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, j.Namespace, rendered.Storage.S3, mdbBackup, takenAt.Time) + status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) } } backup := &backupsv1alpha1.Backup{ From 59f0013b1ff205a62387dafff80349b10e8f660b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 13:21:55 +0000 Subject: [PATCH 18/20] fix(backups): attribute a MariaDB dump only when no other run can own it Every backup of a MariaDB application writes its dump under one shared prefix, so the probe took the dump whose file-name timestamp (the backup pod's clock) and LastModified (the object store's clock) both fall in the operator Backup's lifetime, with a minute of slack for skew. That keeps a neighbour's dump out only while this run's own dump is in the window. With the pod clock more than the slack behind, the own dump is named before the window opens, and a previous run that finished within the minute before this one was created becomes the single match: a 123-byte dump was reported as 666 bytes. A pod clock ahead or an object store clock behind does the same with the next run, and an own dump that is gone by the time it is looked up, as it can be for a restore long after the backup, leaves a neighbour as the single match with no skew at all. A dump is now attributed only when ownership is proven: - it is the one dump that either clock places in the window. A dump that one clock places there and the other does not may be this run's own with a skewed clock, so it leaves any other match unproven, the way a second dump in the window already did, and is not taken itself; - both of its clocks place it in the window; - neither clock places it in the window of another operator Backup that writes under the same bucket and prefix and has completed. This rejects a neighbour's dump when this run's own is missing, which the clocks alone cannot tell. Operator Backups still running are not counted as neighbours: their window has no end, and one stuck short of Complete (a Job whose pod never starts stays JobRunning for good) would otherwise keep every later run of the application from being sized. A dump such a run writes inside this run's window still competes with this run's own. Narrower slack does not prove ownership: a neighbour that finished within the narrower slack still matches, and a smaller skew pushes the own dump out. The Job's start and completion times are stamped by the control plane, like the operator Backup's own, so they narrow the window by seconds without removing the dependence on the two clocks a dump carries. What is left: a neighbour's dump is taken only when this run's own is missing from the listing or misplaced on both clocks and the neighbour's operator Backup no longer exists or has not completed. Every other unproven case leaves the size unset, as two runs that overlap or follow each other within the slack already did. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- internal/backupcontroller/artifact_size.go | 3 +- .../backupcontroller/mariadb_artifact_size.go | 115 ++++++++++--- .../mariadb_artifact_size_test.go | 155 +++++++++++++++++- 3 files changed, 250 insertions(+), 23 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 063e165d6..941b00323 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -60,7 +60,8 @@ type s3Connection struct { // so that the restore path, which has a reconciler of its own, can read // the same objects. type objectStore struct { - // reader must reach Secrets; the manager's client reads them uncached. + // reader reads Secrets, which the manager's client fetches uncached, + // and lists operator Backups, which it serves from the cache. reader client.Reader // newLister overrides the S3 client; nil means a real minio client. newLister func(conn s3Connection) (s3ObjectLister, error) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index d10f1c4a5..6397325ca 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -8,6 +8,10 @@ import ( "strings" "time" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) @@ -60,6 +64,10 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } + others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + if err != nil { + return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) + } conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) if err != nil { return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) @@ -76,35 +84,104 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt if err != nil { return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err) } - return matchMariaDBDump(objects, prefix, mdbBackup.CreationTimestamp.Time, completedAt) + own := newMariaDBRun(mdbBackup, mdbBackup.CreationTimestamp.Time, completedAt) + return matchMariaDBDump(objects, prefix, own, others) } -// matchMariaDBDump picks the dump of the run that lived from created to -// completed. The prefix is shared by every backup of the application, so -// the dump is identified by both the timestamp in its file name and its -// LastModified falling inside that lifetime: the two come from different -// clocks, so a skewed pod clock alone cannot pull a neighbouring run's dump -// into the window. -func matchMariaDBDump(objects []s3ObjectInfo, prefix string, created, completed time.Time) (mariadbDump, error) { - lower := created.Add(-mariadbSizeProbeClockSkew) - upper := completed.Add(mariadbSizeProbeClockSkew) - var matched []mariadbDump +// mariadbRun is the span in which an operator Backup can have written its +// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. +type mariadbRun struct { + name string + lower, upper time.Time +} + +func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { + return mariadbRun{ + name: b.Namespace + "/" + b.Name, + lower: created.Add(-mariadbSizeProbeClockSkew), + upper: completed.Add(mariadbSizeProbeClockSkew), + } +} + +func (r mariadbRun) placesOnBothClocks(d mariadbDump) bool { + return inWindow(d.Timestamp, r.lower, r.upper) && inWindow(d.LastModified, r.lower, r.upper) +} + +func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { + return inWindow(d.Timestamp, r.lower, r.upper) || inWindow(d.LastModified, r.lower, r.upper) +} + +// matchMariaDBDump picks the dump of run own among objects, which share a +// prefix with every other run of the application. A dump carries two +// clocks, the backup pod's in its file name and the object store's in its +// LastModified, and either can be skewed. A dump is taken only when it is +// the one dump either clock places in own's window, both clocks place it +// there, and neither places it in the window of another finished run: a +// dump in the window on one clock only may be this run's own, misplaced by +// a skewed clock, which leaves any other candidate unproven. A neighbour's +// dump is still taken if this run's own is missing from the listing, or +// misplaced on both clocks, and the neighbour's operator Backup is gone or +// has not completed. +func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { + var near []mariadbDump for _, obj := range objects { name := strings.TrimPrefix(obj.Key, prefix) ts, ok := parseMariaDBLogicalBackupFile(name) - if !ok || !inWindow(ts, lower, upper) || !inWindow(obj.LastModified, lower, upper) { + if !ok { continue } - matched = append(matched, mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified}) + d := mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified} + if own.placesOnEitherClock(d) { + near = append(near, d) + } } - switch len(matched) { - case 0: + if len(near) == 0 { return mariadbDump{}, errMariaDBDumpNotFound - case 1: - return matched[0], nil - default: - return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime", errMariaDBDumpAmbiguous, len(matched)) } + if len(near) > 1 { + return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime on at least one clock", errMariaDBDumpAmbiguous, len(near)) + } + d := near[0] + if !own.placesOnBothClocks(d) { + return mariadbDump{}, fmt.Errorf("%w: %s falls in its lifetime on one clock only", errMariaDBDumpAmbiguous, d.Name) + } + for _, other := range others { + if other.placesOnEitherClock(d) { + return mariadbDump{}, fmt.Errorf("%w: %s also falls in the lifetime of operator Backup %s", errMariaDBDumpAmbiguous, d.Name, other.name) + } + } + return d, nil +} + +// finishedMariaDBRuns returns the runs of the other operator Backups that +// write under the same bucket and prefix as b and have completed, whether +// their Job succeeded or failed. One still running is left out: its window +// has no end yet, and an operator Backup stuck short of Complete, as one +// whose Job pod never starts is, would otherwise contest every later run of +// the application for good. The endpoint is not compared, since one has +// several spellings, and a false neighbour only leaves a size unset. +func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { + list := &mariadbtypes.BackupList{} + if err := reader.List(ctx, list); err != nil { + return nil, err + } + var runs []mariadbRun + for i := range list.Items { + other := &list.Items[i] + if other.Namespace == b.Namespace && other.Name == b.Name { + continue + } + s3 := other.Spec.Storage.S3 + if s3 == nil || s3.Bucket != b.Spec.Storage.S3.Bucket || mariadbOperatorS3Prefix(s3.Prefix) != mariadbOperatorS3Prefix(b.Spec.Storage.S3.Prefix) { + continue + } + cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) + if cond == nil || cond.Status != metav1.ConditionTrue { + continue + } + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) + } + return runs, nil } // mariadbS3Connection resolves the Secrets and TLS settings of the operator diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 719678892..9d649c15c 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -239,8 +239,9 @@ func TestMatchMariaDBDump(t *testing.T) { completed := created.Add(3 * time.Minute) const prefix = "tenant/db/" own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)} + run := newMariaDBRun(&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, created, completed) - got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, created, completed) + got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, run, nil) if err != nil { t.Fatalf("matchMariaDBDump: %v", err) } @@ -255,11 +256,11 @@ func TestMatchMariaDBDump(t *testing.T) { t.Errorf("matchMariaDBDump = %+v, want %+v", got, want) } - if _, err := matchMariaDBDump(nil, prefix, created, completed); !errors.Is(err, errMariaDBDumpNotFound) { + if _, err := matchMariaDBDump(nil, prefix, run, nil); !errors.Is(err, errMariaDBDumpNotFound) { t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err) } second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)} - if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, created, completed); !errors.Is(err, errMariaDBDumpAmbiguous) { + if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, run, nil); !errors.Is(err, errMariaDBDumpAmbiguous) { t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err) } } @@ -303,6 +304,154 @@ func TestFindMariaDBDump(t *testing.T) { } } +// TestFindMariaDBDump_Ownership drives runs of one application that share +// a prefix. This run lives from t0 to t0+40s, so its window with the +// one-minute slack is [t0-1m, t0+1m40s]; its own dump is 123 bytes and a +// neighbour's is 666 or 777. A neighbour's dump must never be returned. +func TestFindMariaDBDump_Ownership(t *testing.T) { + t0 := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + const prefix = "tenant/db/" + run := func(name string, created, completed time.Time) *mariadbtypes.Backup { + b := operatorBackupStoringTo(t, s3, created) + b.Name = name + b.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(completed))} + return b + } + dump := func(named, uploaded time.Time, size int64) s3ObjectInfo { + return s3ObjectInfo{Key: dumpKey(prefix, named, "gzip"), Size: size, LastModified: uploaded} + } + own := run("op-bk", t0, t0.Add(40*time.Second)) + find := func(t *testing.T, objects []s3ObjectInfo, operatorBackups ...client.Object) (mariadbDump, error) { + t.Helper() + store := objectStore{ + reader: newMariaDBStrategyTestClient(t, append(operatorBackups, own, s3CredsSecret("tenant", "creds"))...), + newLister: func(s3Connection) (s3ObjectLister, error) { + return &fakeS3Lister{objects: objects}, nil + }, + } + return findMariaDBDump(context.Background(), store, own, t0.Add(40*time.Second)) + } + wantAmbiguous := func(t *testing.T, got mariadbDump, err error) { + t.Helper() + if !errors.Is(err, errMariaDBDumpAmbiguous) { + t.Errorf("findMariaDBDump = %+v, %v; want errMariaDBDumpAmbiguous", got, err) + } + } + wantOwn := func(t *testing.T, got mariadbDump, err error) { + t.Helper() + if err != nil || got.Size != 123 { + t.Errorf("findMariaDBDump = %+v, %v; want this run's 123-byte dump", got, err) + } + } + + t.Run("a pod clock behind does not hand this run the previous run's dump", func(t *testing.T) { + // The previous run finished 20s before this one was created and its + // operator Backup is gone. This run's pod clock is 5m behind, so its + // own dump is named outside the window and uploaded inside it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-30*time.Second), t0.Add(-20*time.Second), 666), + dump(t0.Add(5*time.Second-5*time.Minute), t0.Add(30*time.Second), 123), + }) + wantAmbiguous(t, got, err) + }) + + t.Run("a pod clock ahead does not hand this run the next run's dump", func(t *testing.T) { + // The next run started 5s after this one completed and has uploaded + // its dump by the time this one is probed; it has not completed yet. + next := operatorBackupStoringTo(t, s3, t0.Add(45*time.Second)) + next.Name = "op-next" + next.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(45 * time.Second)), + }} + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(5*time.Second+5*time.Minute), t0.Add(30*time.Second), 123), + dump(t0.Add(50*time.Second), t0.Add(80*time.Second), 777), + }, next) + wantAmbiguous(t, got, err) + }) + + t.Run("an object store clock behind does not hand this run the next run's dump", func(t *testing.T) { + // The store clock is 2m behind: this run's upload is stamped before + // the window opens, the next run's is pulled back into it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(5*time.Second), t0.Add(30*time.Second-2*time.Minute), 123), + dump(t0.Add(50*time.Second), t0.Add(3*time.Minute-2*time.Minute), 777), + }) + wantAmbiguous(t, got, err) + }) + + t.Run("a dump another finished run wrote is not taken when this run's own is gone", func(t *testing.T) { + // This run's dump was pruned; the next run's falls in this run's + // window on both clocks, and in its own. + next := run("op-next", t0.Add(45*time.Second), t0.Add(85*time.Second)) + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(50*time.Second), t0.Add(75*time.Second), 777), + }, next) + wantAmbiguous(t, got, err) + }) + + t.Run("a dump the window holds on one clock only is not taken", func(t *testing.T) { + // This run's dump is gone; the previous run's operator Backup is too, + // and its dump, started 2m before this run, finished uploading 20s + // before it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-2*time.Minute), t0.Add(-20*time.Second), 666), + }) + if err == nil { + t.Errorf("findMariaDBDump = %+v; want no dump attributed", got) + } + }) + + t.Run("a run still in progress does not contest this run's dump", func(t *testing.T) { + // A concurrent run created 30s before this one has not uploaded yet. + running := operatorBackupStoringTo(t, s3, t0.Add(-30*time.Second)) + running.Name = "op-running" + running.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-30 * time.Second)), + }} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, running) + wantOwn(t, got, err) + }) + + t.Run("runs minutes apart each keep their own dump", func(t *testing.T) { + previous := run("op-previous", t0.Add(-4*time.Minute), t0.Add(-3*time.Minute)) + next := run("op-next", t0.Add(3*time.Minute), t0.Add(4*time.Minute)) + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-230*time.Second), t0.Add(-190*time.Second), 666), + dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123), + dump(t0.Add(185*time.Second), t0.Add(220*time.Second), 777), + }, previous, next) + wantOwn(t, got, err) + }) + + t.Run("a run that writes elsewhere does not contest this run's dump", func(t *testing.T) { + other := newRenderedMariaDBTemplate().Storage.S3 + other.Prefix = "tenant/other" + elsewhere := operatorBackupStoringTo(t, other, t0) + elsewhere.Name = "op-elsewhere" + elsewhere.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(t0.Add(40 * time.Second)))} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, elsewhere) + wantOwn(t, got, err) + }) + + t.Run("a run stuck short of Complete does not contest later runs", func(t *testing.T) { + // A backup Job whose pod never starts leaves its operator Backup + // JobRunning for good. + stuck := operatorBackupStoringTo(t, s3, t0.Add(-3*time.Hour)) + stuck.Name = "op-stuck" + stuck.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-3 * time.Hour)), + }} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, stuck) + wantOwn(t, got, err) + }) +} + // TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client // against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and // response parsing are exercised together. From d5559bb934c5c264eff0101fa30a1387e1144692 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 5 Nov 2026 07:56:39 +0000 Subject: [PATCH 19/20] test(backups): remove the Altinity strategy tests' scratch directories Every test in hack/altinity-backup-strategy.bats made a directory with mktemp -d and never removed it, so each passing run of the file left nine directories in TMPDIR, and they pile up on any machine that keeps it between runs. Each test now removes its directory as its last statement. A failed assertion aborts the body before that line, so a failing test keeps its stub log and rendered script for inspection. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- hack/altinity-backup-strategy.bats | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 234ef0b8b..0861da42d 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -105,6 +105,7 @@ remote_rows() { echo "FAIL: render_script reported success with no chart" false fi + rm -rf "$tmp" } @test "restore reads back the archive the Backup recorded, not the newest one" { @@ -116,6 +117,7 @@ remote_rows() { PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] + rm -rf "$tmp" } @test "restore fails naming the recorded archive when remote storage lacks it" { @@ -133,6 +135,7 @@ remote_rows() { echo "FAIL: restored another archive: $(cat "$tmp/curl.log")" false fi + rm -rf "$tmp" } @test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { @@ -150,6 +153,7 @@ remote_rows() { grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' + rm -rf "$tmp" } @test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { @@ -167,6 +171,7 @@ remote_rows() { echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")" false fi + rm -rf "$tmp" } @test "backup writes the archive under the name the controller chose" { @@ -175,6 +180,7 @@ remote_rows() { render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" + rm -rf "$tmp" } @test "a retried backup waits on the attempt the sidecar is still running" { @@ -189,6 +195,7 @@ remote_rows() { echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")" false fi + rm -rf "$tmp" } @test "a retried backup after a failed attempt resumes the same archive" { @@ -198,6 +205,7 @@ remote_rows() { printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions" PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" + rm -rf "$tmp" } @test "backup falls back to a timestamped name when the controller supplies none" { @@ -206,4 +214,5 @@ remote_rows() { render_script "$tmp/s.sh" backup myapp "" "" "" PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log" + rm -rf "$tmp" } From d7ae34dfc3da4c7ef0725f6fc937354df98b64c8 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 5 Nov 2026 07:56:44 +0000 Subject: [PATCH 20/20] fix(backups): fail a ClickHouse BackupJob whose Backup name is taken A Backup outlives the BackupJob that made it, so a BackupJob created again under a name already used meets the earlier run's Backup when its own Job completes. That happens by hand, and from a Plan reconciled again inside its starting deadline, since a Plan names each BackupJob after its schedule slot. The Altinity driver took the earlier Backup as its result: the BackupJob turned Succeeded with a BackupRef recording the earlier run's archive, the archive this run wrote was recorded on nothing, and restoring what the BackupJob reported brought back the older data. An existing Backup is now taken as this run's only when it records this run's archive, which is what an earlier reconcile of the same run leaves when its status write is lost. Any other fails the BackupJob, naming both archives, and leaves the earlier Backup untouched. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../altinitystrategy_controller.go | 7 ++ .../altinitystrategy_controller_test.go | 114 ++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index f7f60bde8..a0868ec38 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -509,6 +509,13 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact( if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { return nil, getErr } + // A Backup outlives its BackupJob, so a BackupJob re-created under a + // used name meets the earlier run's Backup here. Only one recording + // this run's archive was created by an earlier reconcile of this run. + if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive { + return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote", + backup.Namespace, backup.Name, got, archive) + } return existing, nil } return backup, nil diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 7797842c0..e5f1e1695 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -3,6 +3,7 @@ package backupcontroller import ( "context" + "strings" "testing" "time" @@ -861,6 +862,119 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { } } +// TestReconcileAltinity_BackupNameAlreadyTaken covers a completed Job whose +// Backup name is already in use. A Backup outlives its BackupJob, so a +// BackupJob re-created under an earlier run's name finds that run's Backup, +// and claiming it would point the new run at an archive it did not write. +// A Backup recording this run's own archive is the one an earlier reconcile +// created before its status write was lost, and is adopted. +func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { + const thisRun = "clickhouse-ch-test-20260305T050607Z" + cases := []struct { + name string + existing string // archive the existing Backup records; empty for none + wantSucceeded bool + }{ + {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records no archive", existing: ""}, + {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"} + if tc.existing != "" { + driverMD[altinityArchiveMetaKey] = tc.existing + } + existing := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: backupJob.Name, Namespace: backupJob.Namespace}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 22, 0, time.UTC)), + DriverMetadata: driverMD, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, + clientfake.NewClientBuilder().WithObjects(backupJob, strategy, existing)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+thisRun { + t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, thisRun) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + updated := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if tc.wantSucceeded { + if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Errorf("phase = %q (%s), want Succeeded", updated.Status.Phase, updated.Status.Message) + } + if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name != existing.Name { + t.Errorf("BackupRef = %v, want %s", updated.Status.BackupRef, existing.Name) + } + } else { + if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed: the run would claim a Backup recording %q instead of %q", updated.Status.Phase, tc.existing, thisRun) + } + if updated.Status.BackupRef != nil { + t.Errorf("BackupRef = %v, want none", updated.Status.BackupRef) + } + if !strings.Contains(updated.Status.Message, thisRun) { + t.Errorf("failure message %q does not name the archive this run wrote", updated.Status.Message) + } + } + kept := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKeyFromObject(existing), kept); err != nil { + t.Fatalf("get existing Backup: %v", err) + } + if got := kept.Spec.DriverMetadata[altinityArchiveMetaKey]; got != tc.existing { + t.Errorf("existing Backup now records archive %q, want %q left in place", got, tc.existing) + } + }) + } +} + func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) { cases := []struct { name string