mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-10 06:24:09 +09:00
Myasnikov Daniil
a2d9cc766c
Image overrides let a tenant run an arbitrary container in the
management cluster, including helpers with access to the Talos CA
secrets and the tenant cluster's API credentials. Drop those overrides
and run the images the platform ships.
The Kubernetes and KubernetesNodes helpers (the wait-for-kubeconfig
and Proxmox cloud-config init containers, the bootstrap-token and OIDC
RBAC Jobs, the talos-csr-signer sidecar, the Talos reconcile and unpin
Jobs) take the bundled image references through cozy-lib.image, so
they still follow the configured registry. OpenSearch no longer sets
an image and leaves it to the operator, which follows the version the
chart's version map selects; an explicit version: null now fails the
render instead of falling back to v2. The Talos installer settings for
tenant worker VMs stay.
FoundationDB replaces the free-form cluster.version with version, v7.3
or v7.4, mapped to 7.3.63 and 7.4.1; the 7.1 line is no longer
offered. The chart refuses a render where cluster.version and version
name different release lines, which also covers variants without
platform migrations. Migration 60 carries an existing cluster.version
over to its release line and leaves the legacy value in place while
the old chart can still render it. It records what needs attention in
the cozystack.io/migration-60-needs-attention annotation on the
cozystack-version ConfigMap: a FoundationDB release it cannot carry
over or whose patch changes, a release that still sets one of the
removed image overrides, an OpenSearch release storing version: null.
A dry run reports them without patching, annotating or advancing the
stamp. Migration 56 stops carrying the removed kubectl override into
adopted node pools.
OpenSearch and FoundationDB get version-map update scripts that keep
the current default while it is still supported.
Stored image overrides are still accepted and the charts ignore them.
On the first reconcile OpenSearch nodes move to the image of their
selected version and Kubernetes components whose image changed
restart. Air-gapped installations need docker.io and ghcr.io mirrored
on the management-cluster nodes.
Assisted-by: LLM
(cherry picked from commit 749d401899)
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
215 lines
13 KiB
Bash
215 lines
13 KiB
Bash
#!/usr/bin/env bats
|
|
# -----------------------------------------------------------------------------
|
|
# Unit tests for platform migration 60 --> 61 (carry a FoundationDB
|
|
# application's cluster.version over to the version field that replaces it, and
|
|
# record what cannot be carried over).
|
|
#
|
|
# The chart no longer reads cluster.version, so a cluster that set it to a line
|
|
# other than 7.3 would move to the v7.3 default on its first reconcile after the
|
|
# upgrade. These drive the real migration script against a fake kubectl
|
|
# (hack/testdata/migration-60-foundationdb/) and pin that:
|
|
# - each release that set cluster.version on 7.3 or 7.4 gets version for
|
|
# that line, through a JSON patch that tests the values it read and leaves
|
|
# cluster.version in place for the old chart;
|
|
# - releases that never set cluster.version, already have version, or are not
|
|
# FoundationDB are not touched;
|
|
# - an unlabelled release is still found by its chart;
|
|
# - a 7.1 cluster, whose client the operator no longer carries, is recorded
|
|
# rather than carried, and so is a version set to a line no longer offered;
|
|
# - a value outside those lines, a value the apiserver stored as a number, a
|
|
# spec.values that is not an object at all, a carried release whose patch
|
|
# moves, and an OpenSearch, Kubernetes or KubernetesNodes release still
|
|
# setting an image override are recorded and do not block the stamp;
|
|
# - a failed scan or patch aborts before the stamp;
|
|
# - a run retried after an aborted one, which finds version already written,
|
|
# still records the patch the release moves to;
|
|
# - an OpenSearch release storing version: null is recorded;
|
|
# - the tag map the migration carries matches the chart's.
|
|
#
|
|
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
|
|
# own line; there is no bats `run`/`$status`/`setup`/`teardown`. Assertions are
|
|
# direct shell tests that exit non-zero on failure.
|
|
#
|
|
# Run with: hack/cozytest.sh hack/migration-60-foundationdb-version.bats
|
|
# -----------------------------------------------------------------------------
|
|
|
|
load test_helper
|
|
|
|
FAKEBIN="$PWD/hack/testdata/migration-60-foundationdb"
|
|
MIG="$PWD/packages/core/platform/images/migrations/migrations/60"
|
|
|
|
prep() {
|
|
WORK=$(mktemp -d)
|
|
export FAKE_CMDLOG="$WORK/cmdlog"
|
|
export FAKE_HR_LIST="$WORK/hrlist.json"
|
|
: > "$FAKE_CMDLOG"
|
|
export PATH="$FAKEBIN:$PATH"
|
|
export NAMESPACE=cozy-system
|
|
unset FAKE_LIST_FAIL FAKE_PATCH_FAIL MIGRATION_DRY_RUN
|
|
cat > "$FAKE_HR_LIST" <<'JSON'
|
|
{"items":[
|
|
{"metadata":{"namespace":"tenant-a","name":"foundationdb-old71","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-foundationdb-application-default-foundationdb"},
|
|
"values":{"cluster":{"version":"7.1.67","redundancyMode":"double"}}}},
|
|
{"metadata":{"namespace":"tenant-a","name":"foundationdb-new74","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"cluster":{"version":"7.4.3"}}}},
|
|
{"metadata":{"namespace":"tenant-b","name":"foundationdb-pinned73","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"cluster":{"version":"7.3.63"}}}},
|
|
{"metadata":{"namespace":"tenant-b","name":"foundationdb-defaults","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"cluster":{"redundancyMode":"double"}}}},
|
|
{"metadata":{"namespace":"tenant-b","name":"foundationdb-migrated","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"version":"v7.3","cluster":{"version":"7.3.63"}}}},
|
|
{"metadata":{"namespace":"tenant-b","name":"foundationdb-stale-version","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"version":"v7.1","cluster":{"version":"7.1.67"}}}},
|
|
{"metadata":{"namespace":"tenant-c","name":"foundationdb-unlabelled"},
|
|
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-foundationdb-application-default-foundationdb"},
|
|
"values":{"cluster":{"version":"7.3.5"}}}},
|
|
{"metadata":{"namespace":"tenant-c","name":"postgres-db","labels":{"apps.cozystack.io/application.kind":"Postgres"}},
|
|
"spec":{"values":{"cluster":{"version":"7.1.0"}}}},
|
|
{"metadata":{"namespace":"tenant-d","name":"foundationdb-ancient","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"cluster":{"version":"6.3.25"}}}},
|
|
{"metadata":{"namespace":"tenant-d","name":"foundationdb-numeric","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"cluster":{"version":7.3}}}},
|
|
{"metadata":{"namespace":"tenant-d","name":"foundationdb-scalar-values","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":"broken"}},
|
|
{"metadata":{"namespace":"tenant-e","name":"opensearch-logs","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
|
|
"spec":{"values":{"images":{"opensearch":"registry.example.test/opensearch:2.19.6"}}}},
|
|
{"metadata":{"namespace":"tenant-e","name":"opensearch-plain","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
|
|
"spec":{"values":{"images":{"opensearch":""},"replicas":3}}},
|
|
{"metadata":{"namespace":"tenant-f","name":"kubernetes-mirrored","labels":{"apps.cozystack.io/application.kind":"Kubernetes"}},
|
|
"spec":{"values":{"images":{"kubectl":"","talosCsrSigner":"mirror.example.test/talos-csr-signer:1","waitForKubeconfig":"mirror.example.test/busybox:1"}}}},
|
|
{"metadata":{"namespace":"tenant-f","name":"kubernetes-plain","labels":{"apps.cozystack.io/application.kind":"Kubernetes"}},
|
|
"spec":{"values":{"images":{"kubectl":"","talosCsrSigner":"","waitForKubeconfig":""}}}},
|
|
{"metadata":{"namespace":"tenant-e","name":"opensearch-null","labels":{"apps.cozystack.io/application.kind":"OpenSearch"}},
|
|
"spec":{"values":{"version":null}}},
|
|
{"metadata":{"namespace":"tenant-f","name":"kubernetes-nodes-mirrored-md0"},
|
|
"spec":{"chartRef":{"kind":"ExternalArtifact","name":"cozystack-kubernetes-nodes-application-kubevirt-kubernetes-nodes"},
|
|
"values":{"images":{"kubectl":"mirror.example.test/kubectl:1"}}}}
|
|
]}
|
|
JSON
|
|
}
|
|
|
|
@test "carries each supported line over and leaves everything else alone" {
|
|
prep
|
|
rc=0
|
|
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
|
|
cat "$WORK/out"
|
|
cat "$FAKE_CMDLOG"
|
|
[ "$rc" -eq 0 ]
|
|
|
|
grep -qxF 'PATCH tenant-a foundationdb-new74 json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.4.3"}}},{"op":"add","path":"/spec/values/version","value":"v7.4"}]' "$FAKE_CMDLOG"
|
|
grep -qxF 'PATCH tenant-b foundationdb-pinned73 json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.3.63"}}},{"op":"add","path":"/spec/values/version","value":"v7.3"}]' "$FAKE_CMDLOG"
|
|
grep -qxF 'PATCH tenant-c foundationdb-unlabelled json [{"op":"test","path":"/spec/values","value":{"cluster":{"version":"7.3.5"}}},{"op":"add","path":"/spec/values/version","value":"v7.3"}]' "$FAKE_CMDLOG"
|
|
|
|
[ "$(grep -c '^PATCH ' "$FAKE_CMDLOG")" -eq 3 ]
|
|
if grep -q 'PATCH [^ ]* foundationdb-old71 ' "$FAKE_CMDLOG"; then echo "carried a 7.1 cluster"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-stale-version ' "$FAKE_CMDLOG"; then echo "patched a release that already has version"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-defaults ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-defaults '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-migrated ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-migrated '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* postgres-db ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* postgres-db '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-ancient ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-ancient '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* opensearch-plain ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* opensearch-plain '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* opensearch-logs ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* opensearch-logs '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-scalar-values ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-scalar-values '"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* kubernetes' "$FAKE_CMDLOG"; then echo "patched a Kubernetes release"; exit 1; fi
|
|
if grep -q 'PATCH [^ ]* foundationdb-numeric ' "$FAKE_CMDLOG"; then echo "unexpected line matching 'PATCH [^ ]* foundationdb-numeric '"; exit 1; fi
|
|
|
|
annotation=$(grep '^ANNOTATE ' "$FAKE_CMDLOG")
|
|
echo "$annotation" | grep -q 'cozystack.io/migration-60-needs-attention='
|
|
echo "$annotation" | grep -q 'tenant-d/foundationdb-ancient=cluster.version:6.3.25'
|
|
echo "$annotation" | grep -q 'tenant-d/foundationdb-numeric=cluster.version:7.3'
|
|
echo "$annotation" | grep -q 'tenant-d/foundationdb-scalar-values=values-not-an-object'
|
|
echo "$annotation" | grep -q 'tenant-e/opensearch-logs=images.opensearch:registry.example.test/opensearch:2.19.6'
|
|
echo "$annotation" | grep -q 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
|
|
echo "$annotation" | grep -q 'tenant-c/foundationdb-unlabelled=cluster.version:7.3.5->7.3.63'
|
|
echo "$annotation" | grep -q 'tenant-a/foundationdb-old71=cluster.version:7.1.67'
|
|
echo "$annotation" | grep -q 'tenant-b/foundationdb-stale-version=version:v7.1'
|
|
echo "$annotation" | grep -q 'tenant-f/kubernetes-mirrored=images.talosCsrSigner:mirror.example.test/talos-csr-signer:1;images.waitForKubeconfig:mirror.example.test/busybox:1'
|
|
echo "$annotation" | grep -q 'tenant-f/kubernetes-nodes-mirrored-md0=images.kubectl:mirror.example.test/kubectl:1'
|
|
echo "$annotation" | grep -q 'tenant-e/opensearch-null=version:null'
|
|
for quiet in opensearch-plain kubernetes-plain foundationdb-pinned73 foundationdb-migrated; do
|
|
if echo "$annotation" | grep -q "/$quiet="; then echo "recorded $quiet, which needs no attention"; exit 1; fi
|
|
done
|
|
[ "$(tail -n1 "$FAKE_CMDLOG")" = "STAMP 61" ]
|
|
if grep -q '^UNHANDLED' "$FAKE_CMDLOG"; then echo "unexpected line matching '^UNHANDLED'"; exit 1; fi
|
|
rm -rf "$WORK"
|
|
}
|
|
|
|
@test "a failed patch aborts before the stamp" {
|
|
prep
|
|
export FAKE_PATCH_FAIL=1
|
|
rc=0
|
|
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
|
|
cat "$WORK/out"
|
|
[ "$rc" -ne 0 ]
|
|
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
|
|
rm -rf "$WORK"
|
|
}
|
|
|
|
@test "a failed fleet scan aborts before the stamp" {
|
|
prep
|
|
export FAKE_LIST_FAIL=1
|
|
rc=0
|
|
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
|
|
cat "$WORK/out"
|
|
[ "$rc" -ne 0 ]
|
|
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "unexpected line matching '^PATCH'"; exit 1; fi
|
|
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
|
|
rm -rf "$WORK"
|
|
}
|
|
|
|
@test "dry run reports patch changes without writing anything" {
|
|
prep
|
|
export MIGRATION_DRY_RUN=1
|
|
rc=0
|
|
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
|
|
cat "$WORK/out"
|
|
[ "$rc" -eq 0 ]
|
|
grep -q 'would set version=v7.4 on FoundationDB tenant-a/foundationdb-new74' "$WORK/out"
|
|
grep -qxF 'WARN: FoundationDB tenant-a/foundationdb-new74 runs 7.4.3, and v7.4 renders 7.4.1 after the upgrade' "$WORK/out"
|
|
grep -qxF 'WARN: FoundationDB tenant-c/foundationdb-unlabelled runs 7.3.5, and v7.3 renders 7.3.63 after the upgrade' "$WORK/out"
|
|
summary=$(grep '^==> Summary:' "$WORK/out")
|
|
printf '%s\n' "$summary" | grep -qF 'carried=3 '
|
|
printf '%s\n' "$summary" | grep -qF 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
|
|
printf '%s\n' "$summary" | grep -qF 'tenant-c/foundationdb-unlabelled=cluster.version:7.3.5->7.3.63'
|
|
if grep -q 'WARN: FoundationDB tenant-b/foundationdb-pinned73 ' "$WORK/out"; then echo "warned on an unchanged patch"; exit 1; fi
|
|
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "unexpected line matching '^PATCH'"; exit 1; fi
|
|
if grep -q '^ANNOTATE' "$FAKE_CMDLOG"; then echo "unexpected line matching '^ANNOTATE'"; exit 1; fi
|
|
if grep -q '^STAMP' "$FAKE_CMDLOG"; then echo "unexpected line matching '^STAMP'"; exit 1; fi
|
|
rm -rf "$WORK"
|
|
}
|
|
|
|
@test "the tag map matches the chart's versions.yaml" {
|
|
migration=$(sed -n "s/^FOUNDATIONDB_TAGS='\\(.*\\)'$/\\1/p" "$MIG" | jq -S .)
|
|
chart=$(yq -o=json '.' packages/apps/foundationdb/files/versions.yaml | jq -S .)
|
|
echo "migration: $migration"
|
|
echo "chart: $chart"
|
|
[ -n "$migration" ]
|
|
[ "$migration" = "$chart" ]
|
|
}
|
|
|
|
@test "a retry after an aborted pass still records the patch a carried release moves to" {
|
|
prep
|
|
# The state an aborted pass leaves behind: version is already written, so the
|
|
# retry has nothing to carry, but the release still moves from 7.4.3 to 7.4.1.
|
|
cat > "$FAKE_HR_LIST" <<'JSON'
|
|
{"items":[
|
|
{"metadata":{"namespace":"tenant-a","name":"foundationdb-new74","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"version":"v7.4","cluster":{"version":"7.4.3"}}}},
|
|
{"metadata":{"namespace":"tenant-a","name":"foundationdb-steady73","labels":{"apps.cozystack.io/application.kind":"FoundationDB"}},
|
|
"spec":{"values":{"version":"v7.3","cluster":{"version":"7.3.63"}}}}
|
|
]}
|
|
JSON
|
|
rc=0
|
|
bash "$MIG" >"$WORK/out" 2>&1 || rc=$?
|
|
cat "$WORK/out"
|
|
cat "$FAKE_CMDLOG"
|
|
[ "$rc" -eq 0 ]
|
|
if grep -q '^PATCH' "$FAKE_CMDLOG"; then echo "patched a release that already has version"; exit 1; fi
|
|
annotation=$(grep '^ANNOTATE ' "$FAKE_CMDLOG")
|
|
echo "$annotation" | grep -q 'tenant-a/foundationdb-new74=cluster.version:7.4.3->7.4.1'
|
|
if echo "$annotation" | grep -q 'foundationdb-steady73'; then echo "recorded a release that stays on its tag"; exit 1; fi
|
|
[ "$(tail -n1 "$FAKE_CMDLOG")" = "STAMP 61" ]
|
|
rm -rf "$WORK"
|
|
}
|