mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
The Sentinels were left on redis:6.2.6-alpine when the operator moved to the freshworks-oss fork, so that its new compiled-in default could not move them on upgrade. 6.2 is end of life, and every instance now runs a Sentinel one or two major versions behind the Redis it monitors. The rollback risk behind that pin, a newer engine rewriting dump.rdb in a format the older one cannot load, does not apply to Sentinel. It stores no RDB or AOF, only sentinel.conf on an emptyDir that the config-copy init container rewrites from the operator's ConfigMap on every start, so moving back to an older image starts from a clean config. Taking the image from the same version map as the data pods also means one image per node instead of two. The e2e comment that relied on busybox's timeout exit code in the Sentinel container is updated, since that container now carries coreutils. Signed-off-by: Yasar Rahman <yasar.rehman@gmail.com> Assisted-by: LLM
756 lines
31 KiB
YAML
756 lines
31 KiB
YAML
# Chainsaw port of hack/e2e-apps/redis.bats ("Create Redis").
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: redis
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: redis-test
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=redis-test
|
|
tail: 50
|
|
steps:
|
|
- name: create-redis
|
|
try:
|
|
- apply:
|
|
file: redis.yaml
|
|
|
|
- name: wait-helmrelease-ready
|
|
try:
|
|
# 11m: above the 600s install timeout cozystack-api gives this release;
|
|
# see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
|
|
- assert:
|
|
timeout: 11m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: redis-test
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
- name: verify-storage-and-workloads
|
|
try:
|
|
# replicas: 2 provisions two StatefulSet-templated PVCs — verify both.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: redisfailover-persistent-data-rfr-redis-test-0
|
|
status:
|
|
phase: Bound
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: redisfailover-persistent-data-rfr-redis-test-1
|
|
status:
|
|
phase: Bound
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: rfs-redis-test
|
|
status:
|
|
(conditions[?type == 'Available']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: rfr-redis-test
|
|
status:
|
|
readyReplicas: 2
|
|
---
|
|
# TLS on. Everything the unit tests can only render runs live here: the
|
|
# rebuilt operator with the TLS patch, the CRD and ClusterRole patches on its
|
|
# chart, cert-manager issuing the per-release chain, and the CA-extraction
|
|
# controller lifting the trust anchor for the tenant. readyReplicas is the
|
|
# handshake witness: the patched probes dial the pod's own listener with
|
|
# --tls, so a pod that still answers plaintext, or serves a certificate the
|
|
# CA did not sign, never becomes Ready.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: redis-tls
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: redis-secure
|
|
- describe:
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
name: redis-secure-tls
|
|
- describe:
|
|
apiVersion: databases.spotahome.com/v1
|
|
kind: RedisFailover
|
|
name: redis-secure
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=redis-secure
|
|
tail: 50
|
|
# A TLS dial that fails or a Secret write the ClusterRole refuses shows up
|
|
# in the operator's log and nowhere in the tenant namespace.
|
|
- podLogs:
|
|
namespace: cozy-redis-operator
|
|
selector: app.kubernetes.io/name=redis-operator
|
|
tail: 100
|
|
steps:
|
|
- name: create-redis-tls
|
|
try:
|
|
- apply:
|
|
file: redis-tls.yaml
|
|
|
|
- name: wait-helmrelease-ready
|
|
try:
|
|
# Same 11m as redis-test above.
|
|
- assert:
|
|
timeout: 11m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: redis-secure
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
# cert-manager issued the chain the chart declares. Ready on the leaf means
|
|
# <release>.tls holds tls.crt, tls.key and ca.crt; without it the pods have
|
|
# nothing to serve and never start.
|
|
- name: verify-certificates-issued
|
|
try:
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: redis-secure-ca
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: redis-secure-tls
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
|
|
- name: verify-workloads-serve-tls
|
|
try:
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: rfr-redis-secure
|
|
status:
|
|
readyReplicas: 2
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: rfs-redis-secure
|
|
status:
|
|
(conditions[?type == 'Available']):
|
|
- status: "True"
|
|
# The operator's own witness that its ClusterRole patch is in force: the
|
|
# CA-only Secret it publishes needs create on Secrets, which upstream
|
|
# v3.3.5 does not grant. Whether it stays out of the tenant's reach is the
|
|
# next step's business.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure.ca-cert
|
|
type: Opaque
|
|
|
|
# Two paths carry a Secret to a tenant and both are asserted. The dashboard
|
|
# Role grants by name; core.cozystack.io/tenantsecrets serves whatever the
|
|
# lineage webhook stamped tenantresource=true from the ApplicationDefinition
|
|
# selectors, and that label-driven path is the one that would leak a
|
|
# private key if the selector ever named the cert-manager Secrets instead of
|
|
# the projection.
|
|
- name: verify-tenant-ca-projection
|
|
try:
|
|
# The name-granting path, on the deployed object rather than the render:
|
|
# the Role must reach exactly the auth Secret. The unit tests pin the same
|
|
# rule, but only against the template, so a Role that fails to apply or
|
|
# gets widened in the cluster is invisible to them.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: redis-secure-dashboard-resources
|
|
(length(rules[?contains(resources, 'secrets')])): 1
|
|
(rules[?contains(resources, 'secrets')].resourceNames[]):
|
|
- redis-secure-auth
|
|
# Positive control, and it comes first: the negatives below are satisfied
|
|
# by an ABSENT label, which is also what a webhook that stamped nothing
|
|
# this run looks like. The auth Secret is chart-rendered, traces to the
|
|
# Redis CR and is granted by name, so it MUST be stamped.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure-auth
|
|
labels:
|
|
internal.cozystack.io/tenantresource: "true"
|
|
# tenant-ca is stamped by the CA-extraction controller; tenantresource is
|
|
# the lineage webhook's verdict from the redis ApplicationDefinition's
|
|
# matchLabels include. Both true means the whole chain held.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure.tenant-ca
|
|
labels:
|
|
internal.cozystack.io/tenant-ca: "true"
|
|
internal.cozystack.io/tenantresource: "true"
|
|
type: Opaque
|
|
# All three enumerated: the promotion is label-driven, so any one of them
|
|
# acquiring the label leaks. != 'true' rather than == 'false' because the
|
|
# cert-manager Secrets carry no ownerReference (enableCertificateOwnerRef
|
|
# is off) and the webhook never stamps them at all.
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure.ca-tls
|
|
(metadata.labels."internal.cozystack.io/tenantresource" != 'true'): true
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure.tls
|
|
(metadata.labels."internal.cozystack.io/tenantresource" != 'true'): true
|
|
- assert:
|
|
timeout: 2m
|
|
resource:
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: redis-secure.ca-cert
|
|
(metadata.labels."internal.cozystack.io/tenantresource" != 'true'): true
|
|
# No retry around this script. The replica link it reads is settled by
|
|
# none of the asserts above and carries its own bounded wait below. The
|
|
# whole step measured 3.0s on run 34711755303, so link_wait is wide
|
|
# against what it costs today. The reads' 30s caps sum past the 5m op,
|
|
# so they share step_budget instead: no read starts after it, one
|
|
# already running ends within its 5s kill grace of it, and the op sits
|
|
# above both, so a read or the budget is what fails and names itself
|
|
# rather than a ceiling that kills the shell and takes the script's own
|
|
# message with it. hack/chainsaw-op-budgets.bats holds the budget under
|
|
# the op.
|
|
#
|
|
# Every redis-cli below runs under the container's own `timeout`. That
|
|
# inner cap, not the op timeout, is what keeps a dial that never answers
|
|
# from taking the whole suite down: Chainsaw kills the shell when the op
|
|
# expires, but it captures stdout and stderr through pipes, and Go's exec
|
|
# waits for every holder of the write end to close it -- the orphaned
|
|
# kubectl is one, so the run goes silent until the job ceiling instead of
|
|
# failing. Sentinel and data pods run the same Debian-based engine image,
|
|
# whose coreutils /usr/bin/timeout reports 124 on expiry; that is not
|
|
# zero, which is all these checks read.
|
|
#
|
|
# bash for pipefail: base64 -d exits 0 on empty input, so without it a
|
|
# failed Secret read below takes the decoder's status and is reported as
|
|
# the emptiness it left behind. The sandbox's /bin/sh is dash, which
|
|
# rejects `set -o pipefail`; `shell:` is a Script field in the Chainsaw
|
|
# v0.2.15 the sandbox image installs.
|
|
- script:
|
|
timeout: 5m
|
|
shell: bash
|
|
content: |
|
|
set -euo pipefail
|
|
|
|
# Every kubectl below runs under a wall-clock bound. The inner
|
|
# `timeout` on a redis-cli cannot fire if the exec stream itself
|
|
# never establishes, and a kubectl still running holds this script's
|
|
# stdout open, which is what turns a stuck call into a silent run to
|
|
# the job ceiling instead of a failure. A call gets 30s at most and
|
|
# never more than step_budget has left.
|
|
step_budget=240
|
|
step_deadline=$(( $(date +%s) + step_budget ))
|
|
k() {
|
|
k_left=$(( step_deadline - $(date +%s) ))
|
|
if [ "$k_left" -le 0 ]; then
|
|
echo "the ${step_budget}s step budget is spent, so this read did not run: $*" >&2
|
|
return 124
|
|
fi
|
|
[ "$k_left" -le 30 ] || k_left=30
|
|
k_start=$(date +%s)
|
|
k_rc=0
|
|
timeout -k 5 "$k_left" kubectl "$@" || k_rc=$?
|
|
if [ "$k_rc" -ne 0 ] && [ $(( $(date +%s) - k_start )) -ge "$k_left" ]; then
|
|
echo "this read ran into its ${k_left}s bound: $*" >&2
|
|
fi
|
|
return "$k_rc"
|
|
}
|
|
|
|
# Exactly one key, and it is ca.crt. The source carries the CA
|
|
# private key, so any second key is a leak in the making.
|
|
keys=$(k -n "$NAMESPACE" get secret redis-secure.tenant-ca \
|
|
-o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' | sort | tr '\n' ' ')
|
|
if [ "$keys" != "ca.crt " ]; then
|
|
echo "projection must hold exactly ca.crt, holds: $keys" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# It lifted the right certificate: the CA cert-manager wrote to
|
|
# <release>.ca-tls, the source the chart's TenantProjection names.
|
|
# Compared by fingerprint, not raw PEM: the controller re-encodes
|
|
# ca.crt from the DER it parsed, so a header or column-width change
|
|
# at the source would redden a byte comparison of a correct copy.
|
|
src=$(k -n "$NAMESPACE" get secret redis-secure.ca-tls -o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
if [ -z "$src" ]; then
|
|
echo "redis-secure.ca-tls has no ca.crt: the declared source name or key is wrong" >&2
|
|
exit 1
|
|
fi
|
|
proj=$(k -n "$NAMESPACE" get secret redis-secure.tenant-ca -o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
if [ -z "$proj" ]; then
|
|
echo "redis-secure.tenant-ca has no ca.crt: the controller published an empty projection" >&2
|
|
exit 1
|
|
fi
|
|
src_fp=$(echo "$src" | openssl x509 -noout -fingerprint -sha256)
|
|
proj_fp=$(echo "$proj" | openssl x509 -noout -fingerprint -sha256)
|
|
if [ "$src_fp" != "$proj_fp" ]; then
|
|
echo "projection does not carry the release CA certificate" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The source really does hold the private key, or the leak check
|
|
# below would pass while proving nothing. grep reads to EOF here
|
|
# rather than quitting at the match under -q: with pipefail, a
|
|
# kubectl still writing would take SIGPIPE and fail the pipeline.
|
|
if ! k -n "$NAMESPACE" get secret redis-secure.ca-tls \
|
|
-o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -x 'tls.key' >/dev/null; then
|
|
echo "redis-secure.ca-tls no longer carries tls.key: this check has stopped proving anything" >&2
|
|
exit 1
|
|
fi
|
|
if grep -qi 'BEGIN .*PRIVATE KEY' <<<"$proj"; then
|
|
echo "projection carries PEM private key material" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The operator's CA-only copy is key-free too; it is kept from the
|
|
# tenant for what it could become, not for what it holds today.
|
|
cacert_keys=$(k -n "$NAMESPACE" get secret redis-secure.ca-cert \
|
|
-o go-template='{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}' | sort | tr '\n' ' ')
|
|
if [ "$cacert_keys" != "ca.crt " ]; then
|
|
echo "operator CA-only Secret must hold exactly ca.crt, holds: $cacert_keys" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The delivery path the README tells a tenant to use, run as the
|
|
# command the README prints. Everything above reads the raw Secret,
|
|
# which only an admin can do.
|
|
anchor=$(k -n "$NAMESPACE" get tenantsecret redis-secure.tenant-ca \
|
|
-o jsonpath='{.data.ca\.crt}' | base64 -d)
|
|
if ! echo "$anchor" | grep -q 'BEGIN CERTIFICATE'; then
|
|
echo "the trust anchor is not served through core.cozystack.io/tenantsecrets" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# And the negative that gives the positive its meaning: the registry
|
|
# must not serve the key-bearing Secrets, nor the operator's copy.
|
|
# A per-object error is not absence, so the message has to say
|
|
# NotFound: a 403, a timeout or an unserved CRD would otherwise read
|
|
# as proof that the Secret is not served, which is the half of this
|
|
# pair the tenant's isolation rests on.
|
|
for s in redis-secure.ca-tls redis-secure.tls redis-secure.ca-cert; do
|
|
if out=$(k -n "$NAMESPACE" get tenantsecret "$s" 2>&1); then
|
|
echo "$s is being served as a tenant secret" >&2
|
|
exit 1
|
|
elif ! printf '%s' "$out" | grep -q 'NotFound'; then
|
|
echo "cannot determine whether $s is served: $out" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# The anchor the tenant is handed verifies the live server. The pods
|
|
# mount the leaf Secret at /tls, whose ca.crt is the same CA; pin
|
|
# that by fingerprint, then complete a handshake through the
|
|
# service name with it. redis-cli verifies the chain and not the
|
|
# name, so this proves the chain and that the listener answers TLS
|
|
# on that name. It says nothing about the SAN list, which redis-cli
|
|
# never checks; the unit tests pin that.
|
|
pod_fp=$(k -n "$NAMESPACE" exec rfr-redis-secure-0 -c redis -- cat /tls/ca.crt \
|
|
| openssl x509 -noout -fingerprint -sha256)
|
|
if [ "$pod_fp" != "$proj_fp" ]; then
|
|
echo "the CA the pods mount is not the anchor the tenant is given" >&2
|
|
exit 1
|
|
fi
|
|
# pinger is an ACL user the operator writes into every redis.conf it
|
|
# generates, ping-only, and the credentials its own liveness probe
|
|
# uses. It is defined in the operator source, so grepping this tree
|
|
# for it finds nothing.
|
|
pong=$(k -n "$NAMESPACE" exec rfr-redis-secure-0 -c redis -- \
|
|
timeout 15 redis-cli --tls --cacert /tls/ca.crt -h "rfr-redis-secure.$NAMESPACE.svc" -p 6379 \
|
|
--user pinger --pass pingpass --no-auth-warning ping) \
|
|
|| pong='(no reply: redis-cli failed or hit the 15s cap)'
|
|
if [ "$pong" != "PONG" ]; then
|
|
echo "TLS handshake through rfr-redis-secure.$NAMESPACE.svc did not answer PONG: $pong" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# TLS replaces plaintext rather than joining it: the operator writes
|
|
# `port 0` beside the TLS port, and every check above would stay
|
|
# green if it stopped doing so. The mirror of the flip case: here the
|
|
# plaintext dial is the one that must not answer.
|
|
#
|
|
# The verdict is the listener's refusal, not the absence of PONG. A
|
|
# missing pod, a 403 on exec, a wrong container name and the k()
|
|
# cap firing print no PONG either, and a check that only looked for
|
|
# it would pass on every one of them. A TLS listener drops a
|
|
# plaintext client at the first bytes of AUTH, so redis-cli reports
|
|
# the AUTH reply as "I/O error", retries once, and exits 1, followed
|
|
# by "Error: Connection reset by peer". The I/O error line is the
|
|
# refusal. A
|
|
# refused connect is not, because nothing listening contradicts the
|
|
# TLS dial above. Nor is the cap expiring: the flip case reads
|
|
# expiry as its refusal because a plaintext listener holds a
|
|
# ClientHello in silence, whereas a TLS listener answers a plaintext
|
|
# dial at once, so silence here is a state this check did not read.
|
|
# Both fail as such, with what the dial printed.
|
|
rc=0
|
|
pong=$(k -n "$NAMESPACE" exec rfr-redis-secure-0 -c redis -- \
|
|
timeout 15 redis-cli -h "rfr-redis-secure.$NAMESPACE.svc" -p 6379 \
|
|
--user pinger --pass pingpass --no-auth-warning ping 2>&1) || rc=$?
|
|
case "$pong" in
|
|
*PONG*)
|
|
echo "the plaintext listener still answers on an instance with TLS on" >&2
|
|
exit 1 ;;
|
|
*"I/O error"*) ;;
|
|
*)
|
|
echo "cannot determine whether the plaintext listener is off (exit $rc): $pong" >&2
|
|
exit 1 ;;
|
|
esac
|
|
|
|
# readyReplicas above witnesses nothing about the link. Redis reports
|
|
# master_link_status:up only in REPL_STATE_CONNECTED, while the
|
|
# operator's readiness script passes a replica on two other
|
|
# predicates -- master_host off 127.0.0.1, and sync not in the
|
|
# transfer phase -- so the whole handshake between them is ready and
|
|
# unlinked, and a dropped link re-enters it. The topology is thus in
|
|
# motion here and one sampling would decide on it. Wait for it to
|
|
# settle, then assert the sentinels moved off the bootstrap address,
|
|
# which nothing above covers at all.
|
|
link_wait=60
|
|
started=$(date +%s)
|
|
deadline=$(( started + link_wait ))
|
|
while :; do
|
|
masters=0
|
|
linked=0
|
|
seen=
|
|
for i in 0 1; do
|
|
rc=0
|
|
info=$(k -n "$NAMESPACE" exec "rfr-redis-secure-$i" -c redis -- \
|
|
sh -c 'REDISCLI_AUTH="$REDIS_PASSWORD" timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 6379 info replication' 2>&1) || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "INFO replication on rfr-redis-secure-$i failed (exit $rc): $info" >&2
|
|
exit 1
|
|
fi
|
|
case "$info" in
|
|
*role:master*) masters=$((masters + 1)); state=master ;;
|
|
*role:slave*)
|
|
case "$info" in
|
|
*master_link_status:up*) linked=$((linked + 1)); state=linked ;;
|
|
*) state=replicating-no-link ;;
|
|
esac
|
|
;;
|
|
*) state=no-replication-role ;;
|
|
esac
|
|
seen="$seen rfr-redis-secure-$i=$state"
|
|
done
|
|
if [ "$masters" = "1" ] && [ "$linked" = "1" ]; then break; fi
|
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
echo "replication did not settle on one master and one linked replica after $(( $(date +%s) - started ))s:$seen" >&2
|
|
exit 1
|
|
fi
|
|
# TODO(e2e-replace-fixed-timeouts): nothing models the link -- the
|
|
# readiness probe does not read master_link_status, and the
|
|
# RedisFailover CRD defines no status at all, so there is no
|
|
# condition to assert on instead of this poll.
|
|
sleep 5
|
|
done
|
|
|
|
sentinel=$(k -n "$NAMESPACE" get pod \
|
|
-l app.kubernetes.io/component=sentinel,app.kubernetes.io/name=redis-secure \
|
|
-o jsonpath='{.items[0].metadata.name}')
|
|
rc=0
|
|
addr=$(k -n "$NAMESPACE" exec "$sentinel" -c sentinel -- \
|
|
timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 26379 \
|
|
sentinel get-master-addr-by-name mymaster) || rc=$?
|
|
# stderr stays out of $addr, whose first line is parsed below; what
|
|
# the exec wrote there is already in the log above this line.
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "get-master-addr-by-name on $sentinel failed (exit $rc): $addr" >&2
|
|
exit 1
|
|
fi
|
|
addr=$(echo "$addr" | head -n 1 | tr -d '\r')
|
|
case "$addr" in
|
|
''|127.0.0.1) echo "sentinel still points at the bootstrap address: '$addr'" >&2; exit 1 ;;
|
|
esac
|
|
check:
|
|
($error == null): true
|
|
---
|
|
# TLS is chosen at creation. A replica restarted onto the TLS listener cannot
|
|
# replicate from a master still on plaintext, and Sentinels already on TLS
|
|
# would fail over to a stale replica, so the operator's RedisFailover schema
|
|
# rejects a change of tls.enabled. This is the live proof: the rejection
|
|
# reaches the HelmRelease as a failed upgrade, and the running plaintext
|
|
# instance is untouched by it.
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: redis-tls-flip-rejected
|
|
spec:
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
name: redis-flip
|
|
- describe:
|
|
apiVersion: databases.spotahome.com/v1
|
|
kind: RedisFailover
|
|
name: redis-flip
|
|
- podLogs:
|
|
selector: app.kubernetes.io/instance=redis-flip
|
|
tail: 50
|
|
- podLogs:
|
|
namespace: cozy-redis-operator
|
|
selector: app.kubernetes.io/name=redis-operator
|
|
tail: 100
|
|
steps:
|
|
- name: create-redis-plaintext
|
|
try:
|
|
- apply:
|
|
file: redis-flip.yaml
|
|
|
|
- name: wait-plaintext-ready
|
|
try:
|
|
# Same 11m as redis-test above.
|
|
- assert:
|
|
timeout: 11m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: redis-flip
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "True"
|
|
# 5m, not 2m: this assert measured 98s of a 120s budget on run
|
|
# 33608673237, and it is the third RedisFailover the group brings up in
|
|
# one namespace, behind two teardowns. The TLS test's equivalent took 44s.
|
|
- assert:
|
|
timeout: 5m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: rfr-redis-flip
|
|
status:
|
|
readyReplicas: 2
|
|
|
|
# Same object with tls.enabled added: Chainsaw applies it as an update to
|
|
# the Redis it created, so cleanup still owns it.
|
|
- name: flip-tls-on
|
|
try:
|
|
# Helm applies the objects it can before the RedisFailover is refused, so a
|
|
# failed run shows this release with a cert-manager chain and a projection
|
|
# of its own next to the rejection. They are inert without the listener,
|
|
# and Chainsaw's cleanup removes them with the rest.
|
|
- apply:
|
|
file: redis-flip-tls.yaml
|
|
|
|
# The apiserver refuses the RedisFailover change with the schema's own
|
|
# message, Helm's upgrade fails on it, and Flux reports that on the release.
|
|
- name: verify-flip-rejected
|
|
try:
|
|
- assert:
|
|
timeout: 3m
|
|
resource:
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: redis-flip
|
|
status:
|
|
(conditions[?type == 'Ready']):
|
|
- status: "False"
|
|
(length(conditions[?type == 'Ready' && contains(message, 'cannot be enabled or disabled on an existing RedisFailover')])): 1
|
|
# Nothing rolled: the pods still run the revision the plaintext install
|
|
# created, and both listeners are what they were.
|
|
- assert:
|
|
timeout: 1m
|
|
resource:
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: rfr-redis-flip
|
|
status:
|
|
readyReplicas: 2
|
|
(currentRevision == updateRevision): true
|
|
- script:
|
|
timeout: 2m
|
|
content: |
|
|
set -eu
|
|
|
|
# Every kubectl below runs under a wall-clock bound. The inner
|
|
# `timeout` on a redis-cli cannot fire if the exec stream itself
|
|
# never establishes, and a kubectl still running holds this script's
|
|
# stdout open, which is what turns a stuck call into a silent run to
|
|
# the job ceiling instead of a failure.
|
|
k() { timeout 30 kubectl "$@"; }
|
|
# pinger: the operator's own ping-only ACL user, as in the TLS case.
|
|
# Both dials run under the container's `timeout`, for the reason the
|
|
# redis-tls script records above -- and here the TLS one is the whole
|
|
# point: there is no listener to answer it, so without an inner cap
|
|
# it blocks forever and hangs the suite instead of failing it.
|
|
pong=$(k -n "$NAMESPACE" exec rfr-redis-flip-0 -c redis -- \
|
|
timeout 15 redis-cli -h "rfr-redis-flip.$NAMESPACE.svc" -p 6379 \
|
|
--user pinger --pass pingpass --no-auth-warning ping) \
|
|
|| pong='(no reply: redis-cli failed or hit the 15s cap)'
|
|
if [ "$pong" != "PONG" ]; then
|
|
echo "plaintext instance stopped answering after the rejected flip: $pong" >&2
|
|
exit 1
|
|
fi
|
|
# Expiry of that cap IS the refusal being asserted, and its only
|
|
# shape. The ClientHello reaches a plaintext listener as an inline
|
|
# command, which redis scans for a newline with strchr, and the
|
|
# handshake header carries a NUL byte ahead of anything random, so
|
|
# the scan ends before any newline, the server waits for a line
|
|
# that never completes and sends nothing, and redis-cli waits with
|
|
# it until the cap kills it: silent, exit 124. kubectl reports that
|
|
# code in its own trailer,
|
|
# and the trailer is what separates the cap that fired inside the
|
|
# pod from the k() cap firing outside it, since an exec that never
|
|
# reached redis-cli leaves none. So the verdict is that trailer,
|
|
# not the absence of PONG, which a missing pod, a 403 on exec, a
|
|
# wrong container name or a name that does not resolve would
|
|
# produce just as well. Those fail as states this check did not
|
|
# read, with what the dial printed.
|
|
rc=0
|
|
pong=$(k -n "$NAMESPACE" exec rfr-redis-flip-0 -c redis -- \
|
|
timeout 15 redis-cli --tls --insecure -h "rfr-redis-flip.$NAMESPACE.svc" -p 6379 \
|
|
--user pinger --pass pingpass --no-auth-warning ping 2>&1) || rc=$?
|
|
case "$pong" in
|
|
*PONG*)
|
|
echo "a TLS listener appeared on an instance whose flip was rejected" >&2
|
|
exit 1 ;;
|
|
*"command terminated with exit code 124"*) ;;
|
|
*)
|
|
echo "cannot determine whether a TLS listener is up (exit $rc): $pong" >&2
|
|
exit 1 ;;
|
|
esac
|
|
check:
|
|
($error == null): true
|
|
---
|
|
apiVersion: chainsaw.kyverno.io/v1alpha1
|
|
kind: Test
|
|
metadata:
|
|
name: redis-2-backup-roundtrip
|
|
spec:
|
|
# tenant-root, not the suite default tenant-test: the backup Job pushes to the
|
|
# in-cluster seaweedfs in tenant-root, and the e2e tenant-test is Cilium
|
|
# egress-isolated from it. run-all.sh provisions its own Bucket + Redis
|
|
# strategy + BackupClass; the run step overrides S3_ENDPOINT to the in-cluster
|
|
# seaweedfs because the Bucket's advertised external ingress URL is an
|
|
# unroutable placeholder in CI. The .svc FQDN is load-bearing: the driver's
|
|
# curl verifies TLS against the copied seaweedfs CA, whose SANs cover
|
|
# '*.tenant-root.svc', so the 2-label 'seaweedfs-s3.tenant-root' would not
|
|
# match — it must be 'seaweedfs-s3.tenant-root.svc'. It still needs the
|
|
# platform backups stack (backupstrategy-controller + the shipped
|
|
# cozy-default-redis strategy, whose client image the demo reuses).
|
|
namespace: tenant-root
|
|
skipDelete: true
|
|
catch:
|
|
- events: {}
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: BackupJob
|
|
- describe:
|
|
apiVersion: backups.cozystack.io/v1alpha1
|
|
kind: RestoreJob
|
|
- describe:
|
|
apiVersion: strategy.backups.cozystack.io/v1alpha1
|
|
kind: Redis
|
|
name: redis-strategy-default
|
|
- podLogs:
|
|
selector: redis.strategy.backups.cozystack.io/mode
|
|
tail: 200
|
|
steps:
|
|
- name: run
|
|
try:
|
|
# Pre-clean so a stale Succeeded BackupJob from a prior run cannot satisfy
|
|
# the wait instantly. Idempotent; the catch cwd is the suite dir.
|
|
- script:
|
|
timeout: 8m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
NAMESPACE="$NAMESPACE" examples/backups/redis/cleanup.sh
|
|
check:
|
|
($error == null): true
|
|
# Honest round-trip: seed a sentinel key, back it up through this demo's own
|
|
# Bucket, restore it in place, then restore into a copy - asserting the exact
|
|
# bytes survive object storage each time (04 prints the to-copy success).
|
|
# 36m: sized with the 660s bucket HR wait in run-all.sh (convention 6:
|
|
# justified in-line).
|
|
- script:
|
|
timeout: 36m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
[ -x examples/backups/redis/run-all.sh ] || { echo "examples/backups/redis/run-all.sh not found or not executable" >&2; exit 1; }
|
|
NAMESPACE="$NAMESPACE" S3_ENDPOINT="https://seaweedfs-s3.tenant-root.svc:8333" \
|
|
examples/backups/redis/run-all.sh
|
|
check:
|
|
($error == null): true
|
|
(contains($stderr, 'To-copy restore verified')): true
|
|
finally:
|
|
- description: backup-flow cleanup
|
|
script:
|
|
timeout: 8m
|
|
content: |
|
|
set -eu
|
|
cd ../../..
|
|
NAMESPACE="$NAMESPACE" examples/backups/redis/cleanup.sh
|