mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 14:02:56 +09:00
Alexey Artamonov
f02ca8cbc2
The backup examples and e2e suites no longer set passwords. They read the generated ones from <release>-credentials with jq, since a username may contain a dot that jsonpath treats as a path step, and log in as an application user after each restore, so a restore that leaves the credentials unusable fails the run. The MariaDB run also checks that an account created out of band on the source is absent from the copy. The unreachable-target restore step asserts a terminal Failed attributed to the recovery window, since the reason itself is best effort. Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
347 lines
15 KiB
YAML
347 lines
15 KiB
YAML
# Chainsaw port of hack/e2e-apps/postgres.bats ("Create DB PostgreSQL").
|
|
#
|
|
# Every `timeout N sh -ec "until kubectl get ..."` + `kubectl wait` pair from
|
|
# the BATS test collapses into a single assert: chainsaw polls until the
|
|
# resource exists AND matches, or the timeout expires. Failure output is a
|
|
# structured diff of the assertion tree against the live object.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: postgres
|
|
spec:
|
|
# Diagnostics collected automatically when any step fails. The BATS suite
|
|
# only does this in harbor.bats, by hand.
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: postgres-test
|
|
- podLogs:
|
|
selector: cnpg.io/cluster=postgres-test
|
|
tail: 50
|
|
steps:
|
|
- name: create-postgres
|
|
try:
|
|
- apply:
|
|
file: postgres.yaml
|
|
|
|
- name: wait-helmrelease-ready
|
|
try:
|
|
- assert:
|
|
timeout: 6m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: postgres-test
|
|
status:
|
|
# JMESPath filter: condition order in the array does not matter
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
- name: wait-init-job-complete
|
|
try:
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: postgres-test-init-job
|
|
status:
|
|
(conditions[?type == 'Complete']):
|
|
- status: "True"
|
|
|
|
- name: verify-services-and-endpoints
|
|
try:
|
|
# Replaces three `until kubectl get svc ... | grep -q 5432` loops
|
|
# TODO(chainsaw>0.2.15): the (ports[0].port) / (ports[*].port) projection form in
|
|
# these Service asserts is a v0.2.15 workaround for mis-evaluated number-literal
|
|
# filters (ports[?port==`5432`]); (ports[0]) also checks only the first port.
|
|
# Restore the filter form on a chainsaw bump. Same workaround lives in
|
|
# kafka/mariadb/harbor — see "Chainsaw v0.2.15 gotchas" in docs/agents/e2e-testing.md.
|
|
- assert:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: postgres-test-r
|
|
spec:
|
|
(ports[0].port): 5432
|
|
- assert:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: postgres-test-ro
|
|
spec:
|
|
(ports[0].port): 5432
|
|
- assert:
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: postgres-test-rw
|
|
spec:
|
|
(ports[0].port): 5432
|
|
# Replaces `until kubectl get endpoints ... | grep -q '[0-9]'` loops.
|
|
# The read-only endpoint is skipped, same as in the BATS test.
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Endpoints
|
|
metadata:
|
|
name: postgres-test-r
|
|
(subsets[0].addresses[0].ip != null): true
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Endpoints
|
|
metadata:
|
|
name: postgres-test-rw
|
|
(subsets[0].addresses[0].ip != null): true
|
|
# Functional probe — the BATS test never actually talks to the database.
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
pod=$(kubectl -n "$NAMESPACE" get pod \
|
|
-l cnpg.io/cluster=postgres-test,cnpg.io/instanceRole=primary \
|
|
-o name | head -n1)
|
|
kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- \
|
|
psql -U postgres -tAc 'SELECT 1'
|
|
check:
|
|
($error == null): true
|
|
(contains($stdout, '1')): true
|
|
# The init-job is operator-created (not applied by chainsaw), so it is
|
|
# not part of automatic cleanup — delete it explicitly, as BATS does.
|
|
finally:
|
|
- delete:
|
|
ref:
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
name: postgres-test-init-job
|
|
|
|
# The trust anchor a client verifies against, checked against the live
|
|
# operator rather than against our own beliefs about it.
|
|
#
|
|
# This is the only independent witness the CA-extraction feature has. Its unit
|
|
# tests build CNPG's CA Secret from a fixture that encodes the same assumption
|
|
# the shipped ApplicationDefinition encodes ("{{ .release }}-ca", key ca.crt),
|
|
# so the two agree by construction and cannot catch each other being wrong.
|
|
# Only a real CNPG can say whether that name and key are right, and only a real
|
|
# apiserver can say whether the lineage webhook actually promotes the
|
|
# projection to a tenant-readable object.
|
|
- name: verify-tenant-ca-projection
|
|
try:
|
|
# Postgres never opts out of TLS: CNPG mints its CA regardless of the
|
|
# chart's tls flag, so the projection is expected on this plain fixture.
|
|
#
|
|
# The labels are the assertion that matters. tenant-ca is stamped by the
|
|
# controller; tenantresource is stamped by the LINEAGE WEBHOOK, computed
|
|
# from the ApplicationDefinition's spec.secrets — the new matchLabels
|
|
# include. It is "true" only if the whole chain held: definition selects the
|
|
# label, webhook admitted the object, verdict came out positive. Nothing
|
|
# short of a live cluster proves that link.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: postgres-test.tenant-ca
|
|
labels:
|
|
internal.cozystack.io/tenant-ca: "true"
|
|
internal.cozystack.io/tenantresource: "true"
|
|
type: Opaque
|
|
# The projection must be the CA and ONLY the CA. Everything below is
|
|
# deterministic once the assert above has passed, so it runs without retry.
|
|
- script:
|
|
content: |
|
|
set -eu
|
|
|
|
# Exactly one key, and it is ca.crt. Any second key is a leak in the
|
|
# making: the source this was lifted from carries a private key.
|
|
keys=$(kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \
|
|
-o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' | sort | tr '\n' ' ')
|
|
if [ "$keys" != "ca.crt " ]; then
|
|
echo "projection must hold exactly ca.crt, holds: $keys" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# It lifted the right certificate: the projection's ca.crt is CNPG's
|
|
# ca.crt. This is what pins "{{ .release }}-ca" to reality — if CNPG ever
|
|
# renames its CA Secret, the controller finds nothing and this fails.
|
|
#
|
|
# Compare the parsed certificates, not the raw PEM. The controller
|
|
# re-encodes ca.crt from the DER it parsed (certificateChainPEM), so a PEM
|
|
# header line or a column-width change at the source would redden a
|
|
# byte-for-byte comparison of a correct projection. The certificate
|
|
# fingerprint is identity-stable across any re-encoding.
|
|
src=$(kubectl -n "$NAMESPACE" get secret postgres-test-ca -o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
proj=$(kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca -o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
if [ -z "$src" ]; then
|
|
echo "CNPG's postgres-test-ca has no ca.crt — the declared source name or key is wrong" >&2
|
|
exit 1
|
|
fi
|
|
src_fp=$(echo "$src" | openssl x509 -noout -fingerprint -sha256)
|
|
proj_fp=$(echo "$proj" | openssl x509 -noout -fingerprint -sha256)
|
|
if [ "$src_fp" != "$proj_fp" ]; then
|
|
echo "projection does not carry CNPG's CA certificate" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# And the point of the whole exercise: the source carries the CA
|
|
# PRIVATE KEY, the projection must not. Asserting the source really
|
|
# does hold ca.key keeps this honest — if CNPG ever stopped storing it
|
|
# there, this check would otherwise pass while proving nothing.
|
|
if ! kubectl -n "$NAMESPACE" get secret postgres-test-ca \
|
|
-o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'ca.key'; then
|
|
echo "CNPG's CA Secret no longer carries ca.key — this check has stopped proving anything" >&2
|
|
exit 1
|
|
fi
|
|
if kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \
|
|
-o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'ca.key'; then
|
|
echo "projection leaked the CA private key" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Belt and braces on the bytes themselves: no PEM key header of any
|
|
# flavour survived into the tenant-readable object.
|
|
if kubectl -n "$NAMESPACE" get secret postgres-test.tenant-ca \
|
|
-o jsonpath='{.data.ca\.crt}' | base64 -d | grep -qi 'BEGIN .*PRIVATE KEY'; then
|
|
echo "projection carries PEM private key material" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The delivery path the README actually tells a tenant to use, run as
|
|
# the command the README prints. Everything above reads the raw Secret,
|
|
# which only an admin can do — it proves the projection's SHAPE and
|
|
# nothing about whether a tenant can reach it.
|
|
#
|
|
# core.cozystack.io/tenantsecrets serves only objects labelled
|
|
# tenantresource=true, and that label is the lineage webhook's verdict,
|
|
# computed from this app's ApplicationDefinition selectors. So this
|
|
# query 404s unless the whole chain held: the definition selects
|
|
# tenant-ca, the webhook admitted the projection, and the verdict came
|
|
# out positive. It is the one assertion that exercises the registry
|
|
# rather than inspecting its input.
|
|
anchor=$(kubectl -n "$NAMESPACE" get tenantsecret postgres-test.tenant-ca \
|
|
-o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
if ! echo "$anchor" | grep -q 'BEGIN CERTIFICATE'; then
|
|
echo "the trust anchor is not served through core.cozystack.io/tenantsecrets" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# And the negative that gives the positive its meaning: the registry
|
|
# must NOT serve CNPG's key-bearing CA. It carries no tenant-ca label,
|
|
# so the definition does not select it and the webhook marks it
|
|
# tenantresource=false. If this ever starts resolving, the label path
|
|
# has stopped gating on the platform's verdict.
|
|
if kubectl -n "$NAMESPACE" get tenantsecret postgres-test-ca >/dev/null 2>&1; then
|
|
echo "CNPG's key-bearing CA is being served as a tenant secret" >&2
|
|
exit 1
|
|
fi
|
|
check:
|
|
($error == null): true
|
|
# The Postgres CR itself was applied by chainsaw and is deleted (and waited
|
|
# on) automatically during cleanup — no explicit teardown step needed.
|
|
---
|
|
# Full barman-cloud backup -> restore round-trips, driving the validated
|
|
# example scripts (examples/backups/postgres) as the harness so the test and
|
|
# the documented flow can never drift. run-all.sh runs Bucket -> source
|
|
# Postgres + sentinel row -> CNPG strategy + BackupClass -> ad-hoc BackupJob
|
|
# (waits Succeeded) -> empty target Postgres + to-copy RestoreJob (waits
|
|
# Succeeded, asserts the sentinel round-tripped through S3) -> a point-in-time
|
|
# restore between two marker writes (step 45, asserts the pre-marker survived
|
|
# and the post-marker did not) -> a negative unreachable-target restore
|
|
# (step 46, asserts it terminates Failed with a message attributing it to the PITR
|
|
# target rather than hanging or falsely succeeding; the reason-classification
|
|
# itself is unit-tested, being best-effort against a still-archiving source). The
|
|
# in-cluster witness that a barman-cloud
|
|
# plugin backup is taken, restorable, point-in-time-recoverable, and that a
|
|
# bad target is rejected quickly.
|
|
#
|
|
# Runs in tenant-root, not the default tenant-test: the S3 client here is the
|
|
# barman-cloud sidecar INSIDE the Postgres Pod, and the e2e tenant is created
|
|
# isolated (hack/e2e-install-cozystack.bats) — its Cilium egress allowlist
|
|
# blocks a tenant-test Pod from reaching tenant-root's seaweedfs, while a
|
|
# tenant-root Pod reaches it over same-tenant egress. The harness targets the
|
|
# in-cluster endpoint (S3_ENDPOINT below) instead of the BucketInfo external
|
|
# ingress URL, which in CI is an unroutable placeholder, and trusts its
|
|
# self-signed cert by copying the seaweedfs CA secret (seaweedfs-ca-cert,
|
|
# auto-discovered from the cert-manager Certificate) into the per-app
|
|
# endpointCA Secret. (The bucket suite's mc checks lean on the same
|
|
# tenant-root seaweedfs, just from the test runner via port-forward.)
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: postgres-2-backup-roundtrip
|
|
spec:
|
|
namespace: tenant-root
|
|
skipDelete: true
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: postgres-pg-src
|
|
- describe:
|
|
apiVersion: postgresql.cnpg.io/v1
|
|
kind: Cluster
|
|
name: postgres-pg-src
|
|
- describe:
|
|
apiVersion: barmancloud.cnpg.io/v1
|
|
kind: ObjectStore
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: BackupJob
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: RestoreJob
|
|
# The barman-cloud backup/WAL work runs in the cnpg Cluster Pods' sidecar; a
|
|
# failed upload or TLS handshake leaves its error only in that Pod's log.
|
|
- podLogs:
|
|
selector: cnpg.io/cluster=postgres-pg-src
|
|
tail: 200
|
|
steps:
|
|
- name: run
|
|
try:
|
|
# The 40m budget is the whole documented flow: bucket provisioning, a
|
|
# 2-replica source Postgres, the plugin base backup to S3, an empty target
|
|
# app, and THREE sequential purge + re-bootstrap restores against it - the
|
|
# latest-point to-copy restore, the point-in-time restore (step 45), and
|
|
# the negative unreachable-target restore (step 46, which sets a short
|
|
# restoreTimeoutSeconds and must terminate Failed at that deadline, attributed
|
|
# to the PITR target), and the in-place restore (step 50, the
|
|
# regression guard for the archive-vs-recovery serverName collision)
|
|
# (convention 6: justified in-line). The old 25m budget predated the PITR
|
|
# phases; four restores plus the negative phase need the larger window.
|
|
# run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
|
|
# HelmReleases rather than polling out the budget.
|
|
- script:
|
|
timeout: 50m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
[ -x examples/backups/postgres/run-all.sh ] || { echo "postgres backup example scripts not found at examples/backups/postgres" >&2; exit 1; }
|
|
COZY_E2E_BACKUP_PREFLIGHT=1 NAMESPACE="$NAMESPACE" S3_ENDPOINT="https://seaweedfs-s3.tenant-root:8333" \
|
|
examples/backups/postgres/run-all.sh
|
|
check:
|
|
($error == null): true
|
|
finally:
|
|
# Resumes the driver-suspended HelmReleases before deleting the RestoreJob
|
|
# so the apps are not stranded, then removes every resource the harness
|
|
# created (chainsaw did not apply them, so they are not part of automatic
|
|
# cleanup).
|
|
- description: backup-flow cleanup
|
|
script:
|
|
timeout: 8m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
NAMESPACE="$NAMESPACE" examples/backups/postgres/cleanup.sh
|