From e22d84f2c8b15d6604fab05b37ca548063bef4d7 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 8 Jun 2026 16:54:38 +0500 Subject: [PATCH] test(e2e): migrate app suite from BATS to Chainsaw and wire into CI Replaces the per-app BATS E2E suite (hack/e2e-apps/*.bats) with declarative Kyverno Chainsaw tests under hack/e2e-chainsaw/, extending this branch's pilot (postgres + bucket) to all 18 app suites. - Declarative ports assert on status/conditions and concrete fields; the repeated "timeout ... until kubectl get" + "kubectl wait" boilerplate collapses into a single polling assert with structured failure diffs and automatic events/describe/podLogs capture on failure. - Inherently imperative suites keep their logic in script steps: openbao init/unseal, kuberture split-horizon probes, vminstance, the gateway admission/impersonation cases, and kubernetes-latest/previous, which wrap the relocated hack/e2e-chainsaw/_lib/run-kubernetes.sh verbatim. - Gateway tests derive the tenant apex from the namespace host label at runtime, so they are host-independent. CI: the e2e job now runs "chainsaw test hack/e2e-chainsaw/" via a new test-chainsaw target and uploads the JUnit chainsaw-report.xml; the chainsaw binary is added to the e2e-sandbox image. install-cozystack and test-openapi stay BATS. The per-app 3-retry loop is dropped, since assertion polling replaces the fixed-timeout flakiness it papered over. Verified against a dev cluster (DB/app, storage and VM suites pass). Service port assertions use the ports[*].port projection form because Chainsaw v0.2.15 mis-evaluates JMESPath number-literal port filters. Assisted-By: Claude Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 85 +- Makefile | 2 +- docs/agents/overview.md | 4 +- hack/e2e-apps/bucket.bats | 95 --- hack/e2e-apps/clickhouse.bats | 50 -- hack/e2e-apps/etcd.bats | 117 --- hack/e2e-apps/external-dns.bats | 56 -- hack/e2e-apps/foundationdb.bats | 124 --- hack/e2e-apps/gateway.bats | 684 ---------------- hack/e2e-apps/harbor.bats | 93 --- hack/e2e-apps/kafka.bats | 61 -- hack/e2e-apps/kubernetes-latest.bats | 13 - hack/e2e-apps/kubernetes-previous.bats | 9 - hack/e2e-apps/kuberture.bats | 251 ------ hack/e2e-apps/mariadb.bats | 51 -- hack/e2e-apps/mongodb.bats | 43 - hack/e2e-apps/openbao.bats | 64 -- hack/e2e-apps/postgres.bats | 58 -- hack/e2e-apps/qdrant.bats | 30 - hack/e2e-apps/redis.bats | 32 - hack/e2e-apps/vminstance.bats | 117 --- hack/e2e-chainsaw/.chainsaw.yaml | 7 +- hack/e2e-chainsaw/.gitignore | 4 + hack/e2e-chainsaw/README.md | 112 ++- .../_lib}/remediation-guard.sh | 0 .../_lib}/run-kubernetes.sh | 3 +- .../clickhouse/chainsaw-test.yaml | 90 +++ hack/e2e-chainsaw/clickhouse/clickhouse.yaml | 29 + hack/e2e-chainsaw/etcd/chainsaw-test.yaml | 192 +++++ hack/e2e-chainsaw/etcd/etcd-backup.yaml | 26 + hack/e2e-chainsaw/etcd/etcd-basic.yaml | 11 + hack/e2e-chainsaw/etcd/etcd-empty-backup.yaml | 12 + .../external-dns/chainsaw-test.yaml | 107 +++ .../external-dns/externaldns-inmemory.yaml | 8 + .../external-dns/externaldns-prefix.yaml | 9 + .../foundationdb/chainsaw-test.yaml | 166 ++++ .../foundationdb/foundationdb.yaml | 36 + hack/e2e-chainsaw/gateway/chainsaw-test.yaml | 759 ++++++++++++++++++ hack/e2e-chainsaw/gateway/gw-minimal.yaml | 13 + hack/e2e-chainsaw/gateway/gw-route-probe.yaml | 31 + .../gateway/package-attach-probe.yaml | 12 + hack/e2e-chainsaw/harbor/chainsaw-test.yaml | 135 ++++ hack/e2e-chainsaw/harbor/harbor.yaml | 27 + hack/e2e-chainsaw/kafka/chainsaw-test.yaml | 115 +++ hack/e2e-chainsaw/kafka/kafka.yaml | 32 + .../kubernetes-latest/chainsaw-test.yaml | 31 + .../kubernetes-previous/chainsaw-test.yaml | 23 + .../e2e-chainsaw/kuberture/chainsaw-test.yaml | 151 ++++ hack/e2e-chainsaw/kuberture/package.yaml | 26 + hack/e2e-chainsaw/kuberture/probes.yaml | 82 ++ hack/e2e-chainsaw/mariadb/chainsaw-test.yaml | 89 ++ hack/e2e-chainsaw/mariadb/mariadb.yaml | 29 + hack/e2e-chainsaw/mongodb/chainsaw-test.yaml | 62 ++ hack/e2e-chainsaw/mongodb/mongodb.yaml | 20 + hack/e2e-chainsaw/openbao/chainsaw-test.yaml | 118 +++ hack/e2e-chainsaw/openbao/openbao.yaml | 12 + hack/e2e-chainsaw/qdrant/chainsaw-test.yaml | 64 ++ hack/e2e-chainsaw/qdrant/qdrant.yaml | 11 + hack/e2e-chainsaw/redis/chainsaw-test.yaml | 64 ++ hack/e2e-chainsaw/redis/redis.yaml | 12 + .../vminstance/chainsaw-test.yaml | 137 ++++ hack/e2e-chainsaw/vminstance/vmdisk-vmi.yaml | 11 + hack/e2e-chainsaw/vminstance/vmdisk.yaml | 11 + hack/e2e-chainsaw/vminstance/vminstance.yaml | 28 + hack/remediation-guard.bats | 18 +- hack/select-e2e.sh | 63 +- hack/select-e2e_test.bats | 25 +- packages/core/testing/Makefile | 16 +- .../testing/images/e2e-sandbox/Dockerfile | 4 + 69 files changed, 2981 insertions(+), 2101 deletions(-) delete mode 100644 hack/e2e-apps/bucket.bats delete mode 100644 hack/e2e-apps/clickhouse.bats delete mode 100644 hack/e2e-apps/etcd.bats delete mode 100644 hack/e2e-apps/external-dns.bats delete mode 100644 hack/e2e-apps/foundationdb.bats delete mode 100644 hack/e2e-apps/gateway.bats delete mode 100644 hack/e2e-apps/harbor.bats delete mode 100644 hack/e2e-apps/kafka.bats delete mode 100644 hack/e2e-apps/kubernetes-latest.bats delete mode 100644 hack/e2e-apps/kubernetes-previous.bats delete mode 100644 hack/e2e-apps/kuberture.bats delete mode 100644 hack/e2e-apps/mariadb.bats delete mode 100644 hack/e2e-apps/mongodb.bats delete mode 100644 hack/e2e-apps/openbao.bats delete mode 100644 hack/e2e-apps/postgres.bats delete mode 100755 hack/e2e-apps/qdrant.bats delete mode 100644 hack/e2e-apps/redis.bats delete mode 100644 hack/e2e-apps/vminstance.bats create mode 100644 hack/e2e-chainsaw/.gitignore rename hack/{e2e-apps => e2e-chainsaw/_lib}/remediation-guard.sh (100%) rename hack/{e2e-apps => e2e-chainsaw/_lib}/run-kubernetes.sh (99%) create mode 100644 hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/clickhouse/clickhouse.yaml create mode 100644 hack/e2e-chainsaw/etcd/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/etcd/etcd-backup.yaml create mode 100644 hack/e2e-chainsaw/etcd/etcd-basic.yaml create mode 100644 hack/e2e-chainsaw/etcd/etcd-empty-backup.yaml create mode 100644 hack/e2e-chainsaw/external-dns/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/external-dns/externaldns-inmemory.yaml create mode 100644 hack/e2e-chainsaw/external-dns/externaldns-prefix.yaml create mode 100644 hack/e2e-chainsaw/foundationdb/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/foundationdb/foundationdb.yaml create mode 100644 hack/e2e-chainsaw/gateway/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/gateway/gw-minimal.yaml create mode 100644 hack/e2e-chainsaw/gateway/gw-route-probe.yaml create mode 100644 hack/e2e-chainsaw/gateway/package-attach-probe.yaml create mode 100644 hack/e2e-chainsaw/harbor/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/harbor/harbor.yaml create mode 100644 hack/e2e-chainsaw/kafka/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/kafka/kafka.yaml create mode 100644 hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/kuberture/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/kuberture/package.yaml create mode 100644 hack/e2e-chainsaw/kuberture/probes.yaml create mode 100644 hack/e2e-chainsaw/mariadb/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/mariadb/mariadb.yaml create mode 100644 hack/e2e-chainsaw/mongodb/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/mongodb/mongodb.yaml create mode 100644 hack/e2e-chainsaw/openbao/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/openbao/openbao.yaml create mode 100644 hack/e2e-chainsaw/qdrant/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/qdrant/qdrant.yaml create mode 100644 hack/e2e-chainsaw/redis/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/redis/redis.yaml create mode 100644 hack/e2e-chainsaw/vminstance/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/vminstance/vmdisk-vmi.yaml create mode 100644 hack/e2e-chainsaw/vminstance/vmdisk.yaml create mode 100644 hack/e2e-chainsaw/vminstance/vminstance.yaml diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 52faceb15..d05a225e8 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -328,10 +328,17 @@ jobs: cd /tmp/$SANDBOX_NAME make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi - # Run E2E tests. With `full-e2e` label the selector is skipped, its - # outputs are empty, the != 'true' guard lets this step run, and the - # full bats list is used. Without the label, the selector decides; - # an empty selection (docs-only) skips this step entirely. + # ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis) + # + # The "Select E2E tests" step above picks the suites impacted by the PR + # diff. With the `full-e2e` label that step is skipped, SELECTED_APPS is + # empty, and we fall back to the whole suite; an empty selection on a + # docs-only PR skips this step entirely (the skip != 'true' guard). + # + # Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls + # every assertion to its own timeout, so the old per-app 3-retry loop is + # gone — a failure is reported as a structured assertion diff, with + # cluster-state diagnostics dumped below on a non-zero exit. - name: Run E2E tests id: e2e_tests if: ${{ steps.select.outputs.skip != 'true' }} @@ -341,40 +348,50 @@ jobs: run: | cd /tmp/$SANDBOX_NAME if [ "$FULL_E2E" = "true" ] || [ -z "$SELECTED_APPS" ]; then - apps_list=$(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1) + suites="" # empty => test-chainsaw runs the whole suite + echo "Running full chainsaw suite" else - apps_list="$SELECTED_APPS" + suites="$SELECTED_APPS" + echo "Running selected chainsaw suites: $suites" fi - failed_tests="" - for app in $apps_list; do - echo "::group::Testing $app" - if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then - echo "::endgroup::" - echo "✅ Test $app completed successfully" - else - echo "::endgroup::" - echo "❌ Test $app failed (no retry — see diagnostics below)" - failed_tests="$failed_tests $app" - echo "::group::Diagnostics for $app" - docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true - docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true - # COSI state at the moment of failure — the CRDs ship no printer - # columns, so pull the readiness fields explicitly (cozyreport - # captures the same at end-of-job, but bucket states move between - # a test failing and the report being taken). - docker exec "$SANDBOX_NAME" sh -c ' - kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1 - kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1 - kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1 - ' || true - echo "::endgroup::" - fi - done - if [ -n "$failed_tests" ]; then - echo "❌ Failed tests:$failed_tests" + if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then + echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)" + echo "::group::Diagnostics: HelmRelease status" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true + echo "::endgroup::" + echo "::group::Diagnostics: recent events" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true + echo "::endgroup::" + # COSI state at the moment of failure — the CRDs ship no printer + # columns, so pull the readiness fields explicitly (cozyreport + # captures the same at end-of-job, but bucket states move between + # a test failing and the report being taken). + echo "::group::Diagnostics: COSI bucket state" + docker exec "$SANDBOX_NAME" sh -c ' + kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1 + kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1 + kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1 + ' || true + echo "::endgroup::" exit 1 fi - echo "✅ All E2E tests passed" + echo "✅ All selected E2E tests passed" + + - name: Collect chainsaw report + if: always() + run: | + cd /tmp/$SANDBOX_NAME + mkdir -p _out + docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \ + _out/chainsaw-report.xml || true + + - name: Upload chainsaw report + if: always() + uses: actions/upload-artifact@v4 + with: + name: chainsaw-report + path: /tmp/${{ env.SANDBOX_NAME }}/_out/chainsaw-report.xml + if-no-files-found: ignore # ▸ Collect debug information (always runs) - name: Collect report diff --git a/Makefile b/Makefile index 78398dbc4..f5db15787 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ assets-cozypkg-%: test: make -C packages/core/testing apply - make -C packages/core/testing test + make -C packages/core/testing e2e unit-tests: helm-unit-tests bats-unit-tests go-unit-tests rd-presets-check test-check-readiness diff --git a/docs/agents/overview.md b/docs/agents/overview.md index f09bdc0a2..0986654b6 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -25,7 +25,7 @@ The platform exposes infrastructure services via the Kubernetes API with ready-m │ └── extra/ # Tenant-specific modules, singleton charts which are used as dependencies ├── dashboards/ # Grafana dashboards for monitoring ├── hack/ # Helper scripts for local development -│ └── e2e-apps/ # End-to-end application tests +│ └── e2e-chainsaw/ # End-to-end application tests (Kyverno Chainsaw) ├── scripts/ # Scripts used by cozystack container │ └── migrations/ # Version migration scripts ├── docs/ # Documentation @@ -101,7 +101,7 @@ Package `values.yaml` files carry annotations (`@param`, `@typedef`, `@field`, ` ## Testing - **Helm unit tests:** `make helm-unit-tests` (runs `hack/helm-unit-tests.sh` over every package that defines a `test` target). `make unit-tests` runs the full unit suite — Helm, BATS, Go, and the preset/readiness checks. -- **E2E tests:** BATS suites in `hack/e2e-apps/` (one `.bats` per app), run through `hack/cozytest.sh`. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md). +- **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md). - **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers. ## Conventions diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats deleted file mode 100644 index 9c885b4d4..000000000 --- a/hack/e2e-apps/bucket.bats +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bats - -@test "Create and Verify Seeweedfs Bucket" { - # Create the bucket resource with readwrite and readonly users - name='test' - kubectl -n tenant-test delete bucket.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - pkill -f "port-forward.*8333:" 2>/dev/null || true - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - # The COSI controller requeues the BucketClaim until the backend Bucket's - # readiness propagates, so this converges within tens of seconds. Assert - # bucketReady=true explicitly: a bare jsonpath match also matches the literal - # "false", so the unqualified form passed against an unready claim. The tight - # bound makes a propagation-race regression fail fast instead of hiding. - kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { - echo "=== BucketClaim did not converge to bucketReady=true ===" - kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} -o yaml 2>&1 || true - echo "=== backend Buckets (cluster-scoped) ===" - kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true - echo "=== objectstorage-controller ===" - kubectl -n cozy-objectstorage-controller get pods 2>&1 || true - false - } - timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}'=true - timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}'=true - - # Get admin (readwrite) credentials - kubectl -n tenant-test get secret bucket-${name}-admin -ojsonpath='{.data.BucketInfo}' | base64 -d > bucket-admin-credentials.json - ADMIN_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' bucket-admin-credentials.json) - ADMIN_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' bucket-admin-credentials.json) - BUCKET_NAME=$(jq -r '.spec.bucketName' bucket-admin-credentials.json) - - # Get viewer (readonly) credentials - kubectl -n tenant-test get secret bucket-${name}-viewer -ojsonpath='{.data.BucketInfo}' | base64 -d > bucket-viewer-credentials.json - VIEWER_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' bucket-viewer-credentials.json) - VIEWER_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' bucket-viewer-credentials.json) - - # Start port-forwarding in this shell (not via `bash -c '... &'` — the - # wrapper subshell exits immediately, the backgrounded kubectl becomes an - # orphan and gets reaped before the first mc S3 request, so `mc cp` sees - # `dial tcp 127.0.0.1:8333: connect: connection refused` even though the - # earlier `nc -z` probe succeeded). The forked port-forward inherits the - # test process group; cozytest.sh kills the whole group when the test - # function returns, so no explicit cleanup is needed. - kubectl port-forward service/seaweedfs-s3 -n tenant-root 8333:8333 >/dev/null 2>&1 & - - # Wait for port-forward to be ready - timeout 30 sh -ec 'until nc -z 127.0.0.1 8333; do sleep 1; done' - - # --- Test readwrite user (admin) --- - mc alias set rw-user https://127.0.0.1:8333 $ADMIN_ACCESS_KEY $ADMIN_SECRET_KEY --insecure - - # Admin can upload - echo "readwrite test" > /tmp/rw-test.txt - mc cp --insecure /tmp/rw-test.txt rw-user/$BUCKET_NAME/rw-test.txt - - # Admin can list - mc ls --insecure rw-user/$BUCKET_NAME/rw-test.txt - - # Admin can download - mc cp --insecure rw-user/$BUCKET_NAME/rw-test.txt /tmp/rw-test-download.txt - - # --- Test readonly user (viewer) --- - mc alias set ro-user https://127.0.0.1:8333 $VIEWER_ACCESS_KEY $VIEWER_SECRET_KEY --insecure - - # Viewer can list - mc ls --insecure ro-user/$BUCKET_NAME/rw-test.txt - - # Viewer can download - mc cp --insecure ro-user/$BUCKET_NAME/rw-test.txt /tmp/ro-test-download.txt - - # Viewer cannot upload (must fail with Access Denied) - echo "readonly test" > /tmp/ro-test.txt - ! mc cp --insecure /tmp/ro-test.txt ro-user/$BUCKET_NAME/ro-test.txt - - # --- Cleanup --- - mc rm --insecure rw-user/$BUCKET_NAME/rw-test.txt - kubectl -n tenant-test delete bucket.apps.cozystack.io ${name} -} diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats deleted file mode 100644 index 72400867a..000000000 --- a/hack/e2e-apps/clickhouse.bats +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bats - -@test "Create DB ClickHouse" { - name='test' - kubectl -n tenant-test delete clickhouse.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr clickhouse-$name --timeout=5m --for=condition=ready - timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1 - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get sts chi-clickhouse-$name-clickhouse-0-1 ; do sleep 10; done" - kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-1 --timeout=140s --for=jsonpath='{.status.replicas}'=1 - kubectl -n tenant-test delete clickhouse $name -} diff --git a/hack/e2e-apps/etcd.bats b/hack/e2e-apps/etcd.bats deleted file mode 100644 index c831f0102..000000000 --- a/hack/e2e-apps/etcd.bats +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bats - -# The etcd chart pins the Helm release name to 'etcd' via -# packages/extra/etcd/templates/check-release-name.yaml, so every test -# must apply its Etcd with metadata.name: etcd. setup() clears any -# prior run so tests remain independent despite the singleton name. - -setup() { - kubectl -n tenant-test delete etcd.apps.cozystack.io --all --ignore-not-found --timeout=2m - # HelmRelease teardown is async relative to the CR deletion above; wait for - # downstream resources so the next test starts from a clean state. - kubectl -n tenant-test wait hr/etcd --for=delete --timeout=2m --ignore-not-found - kubectl -n tenant-test wait secret/etcd-s3-creds --for=delete --timeout=1m --ignore-not-found - kubectl -n tenant-test wait etcdbackupschedule.etcd.aenix.io/etcd --for=delete --timeout=1m --ignore-not-found -} - -dump_diagnostics() { - echo "# --- diagnostics ---" >&3 - kubectl -n tenant-test get etcdcluster,etcdbackupschedule,cronjob -o wide >&3 2>&1 || true - kubectl -n tenant-test describe etcdbackupschedule etcd >&3 2>&1 || true - kubectl -n cozy-etcd-operator logs -l app.kubernetes.io/name=etcd-operator --tail=100 >&3 2>&1 || true -} - -# Wait until the etcd HelmRelease is reconciled by Flux and its -# condition=ready is set. kubectl wait fails immediately on a missing -# resource, so poll for existence first. -wait_etcd_hr_ready() { - timeout 60 sh -ec 'until kubectl -n tenant-test get hr/etcd >/dev/null 2>&1; do sleep 2; done' - kubectl -n tenant-test wait hr/etcd --timeout=5m --for=condition=ready -} - -@test "Create Etcd" { - kubectl apply -f- </dev/null - ! kubectl -n tenant-test get secret etcd-s3-creds 2>/dev/null -} - -@test "Create Etcd with backup schedule" { - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" || { dump_diagnostics; false; } - # Verify the region field propagated to the EtcdBackupSchedule. - REGION=$(kubectl -n tenant-test get etcdbackupschedule.etcd.aenix.io etcd -o jsonpath='{.spec.destination.s3.region}') - [ "$REGION" = "us-east-1" ] - kubectl -n tenant-test get secret etcd-s3-creds -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d | grep -q '^e2e-access-key$' - # The etcd-operator generates a CronJob from the EtcdBackupSchedule. Wait for it. - timeout 120 sh -ec "until [ \"\$(kubectl -n tenant-test get cronjob -l etcd.aenix.io/etcdbackupschedule-name=etcd -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)\" != '' ]; do sleep 5; done" || { dump_diagnostics; false; } -} diff --git a/hack/e2e-apps/external-dns.bats b/hack/e2e-apps/external-dns.bats deleted file mode 100644 index 23b3e8cf6..000000000 --- a/hack/e2e-apps/external-dns.bats +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bats - -teardown() { - kubectl -n tenant-test delete externaldns.apps.cozystack.io --all --ignore-not-found 2>/dev/null || true -} - -@test "Create and Verify ExternalDNS with inmemory provider" { - name='test' - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr ${name} --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name}-system >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr ${name}-system --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" - - kubectl -n tenant-test delete externaldns.apps.cozystack.io ${name} -} - -@test "Create and Verify ExternalDNS with custom annotationPrefix" { - name='test-prefix' - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr ${name} --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name}-system >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr ${name}-system --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done" - - kubectl -n tenant-test delete externaldns.apps.cozystack.io ${name} -} diff --git a/hack/e2e-apps/foundationdb.bats b/hack/e2e-apps/foundationdb.bats deleted file mode 100644 index accfd8e1d..000000000 --- a/hack/e2e-apps/foundationdb.bats +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bats - -@test "Create DB FoundationDB" { - name='test' - kubectl -n tenant-test delete foundationdb.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - - # Wait for HelmRelease to be ready - kubectl -n tenant-test wait hr foundationdb-$name --timeout=300s --for=condition=ready - - # Wait for FoundationDBCluster to be created (name has foundationdb- prefix) - timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name; do sleep 15; done" - - # Wait for cluster to become available (initial reconciliation takes time - allow 5 minutes) - timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.usable_regions}' | grep -q '1'; do sleep 30; done" - - # Check that storage processes are running - timeout 300 sh -ec "until [ \$(kubectl -n tenant-test get pods -l foundationdb.org/fdb-cluster-name=foundationdb-$name,foundationdb.org/fdb-process-class=storage --field-selector=status.phase=Running --no-headers | wc -l) -eq 3 ]; do sleep 15; done" - - # Check that log processes are running (these are the stateless processes) - timeout 300 sh -ec "until [ \$(kubectl -n tenant-test get pods -l foundationdb.org/fdb-cluster-name=foundationdb-$name,foundationdb.org/fdb-process-class=log --field-selector=status.phase=Running --no-headers | wc -l) -ge 1 ]; do sleep 15; done" - - # Check that cluster controller is running - timeout 300 sh -ec "until [ \$(kubectl -n tenant-test get pods -l foundationdb.org/fdb-cluster-name=foundationdb-$name,foundationdb.org/fdb-process-class=cluster_controller --field-selector=status.phase=Running --no-headers | wc -l) -eq 1 ]; do sleep 15; done" - - # Check WorkloadMonitor is created and configured - timeout 120 sh -ec "until kubectl -n tenant-test get workloadmonitor foundationdb-$name; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get workloadmonitor foundationdb-$name -o jsonpath='{.spec.replicas}' | grep -q '3'; do sleep 5; done" - - # Check dashboard resource map is created - kubectl -n tenant-test get configmap foundationdb-$name-resourcemap - - # Verify cluster is healthy (check cluster status) - allow extra time for initial setup - timeout 300 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.available}' | grep -q 'true'; do sleep 20; done" - - # Validate status.configured field - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.configured}' | grep -q 'true'; do sleep 10; done" - - # Validate status.connectionString field exists and contains expected format - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.connectionString}' | grep -q '@.*\.svc\.cozy\.local'; do sleep 10; done" - - # Validate comprehensive status.databaseConfiguration fields - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.logs}' | grep -q '3'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.proxies}' | grep -q '3'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.redundancy_mode}' | grep -q 'double'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.resolvers}' | grep -q '1'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.storage_engine}' | grep -q 'ssd-2'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.databaseConfiguration.usable_regions}' | grep -q '1'; do sleep 10; done" - - # Validate status.desiredProcessGroups field - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.desiredProcessGroups}' | grep -q '^[0-9][0-9]*$'; do sleep 10; done" - - # Validate status.generations.reconciled field - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.generations.reconciled}' | grep -q '^[0-9][0-9]*$'; do sleep 10; done" - - # Validate status.hasListenIPsForAllPods field - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.hasListenIPsForAllPods}' | grep -q 'true'; do sleep 10; done" - - # Validate comprehensive status.health fields - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.fullReplication}' | grep -q 'true'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.status.health.healthy}' | grep -q 'true'; do sleep 10; done" - - # Verify security context is applied correctly (non-root user) - storage_pod=$(kubectl -n tenant-test get pods -l foundationdb.org/fdb-cluster-name=foundationdb-$name,foundationdb.org/fdb-process-class=storage --no-headers | head -n1 | awk '{print $1}') - kubectl -n tenant-test get pod "$storage_pod" -o jsonpath='{.spec.containers[0].securityContext.runAsUser}' | grep -q '4059' - kubectl -n tenant-test get pod "$storage_pod" -o jsonpath='{.spec.containers[0].securityContext.runAsGroup}' | grep -q '4059' - - # Verify volumeClaimTemplate is properly configured in FoundationDBCluster CRD - timeout 60 sh -ec "until kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name -o jsonpath='{.spec.processes.general.volumeClaimTemplate.spec.resources.requests.storage}' | grep -q '1Gi'; do sleep 10; done" - - # Verify PVCs are created with correct storage size (1Gi as specified in test) - timeout 120 sh -ec "until [ \$(kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name --no-headers | wc -l) -ge 3 ]; do sleep 10; done" - kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name -o jsonpath='{.items[*].spec.resources.requests.storage}' | grep -q '1Gi' - - # Verify actual PVC storage capacity matches requested size - kubectl -n tenant-test get pvc -l foundationdb.org/fdb-cluster-name=foundationdb-$name -o jsonpath='{.items[*].status.capacity.storage}' | grep -q '1Gi' - - # Clean up - kubectl -n tenant-test delete foundationdb $name - - # Wait for cleanup to complete - timeout 120 sh -ec "while kubectl -n tenant-test get foundationdbclusters.apps.foundationdb.org foundationdb-$name 2>/dev/null; do sleep 10; done" -} \ No newline at end of file diff --git a/hack/e2e-apps/gateway.bats b/hack/e2e-apps/gateway.bats deleted file mode 100644 index 6ef8bdcf1..000000000 --- a/hack/e2e-apps/gateway.bats +++ /dev/null @@ -1,684 +0,0 @@ -#!/usr/bin/env bats - -@test "Gateway API CRDs are installed and the cilium GatewayClass is Accepted" { - # Gateway API CRDs must exist — installed by packages/system/gateway-api-crds - kubectl wait crd/gatewayclasses.gateway.networking.k8s.io --for=condition=Established --timeout=60s - kubectl wait crd/gateways.gateway.networking.k8s.io --for=condition=Established --timeout=60s - kubectl wait crd/httproutes.gateway.networking.k8s.io --for=condition=Established --timeout=60s - - # Cilium must have registered its built-in GatewayClass once gatewayAPI.enabled - # is true in the cilium values. This verifies the flip in - # packages/system/cilium/values.yaml propagated end-to-end. - timeout 120 sh -ec 'until kubectl get gatewayclass cilium >/dev/null 2>&1; do sleep 2; done' - kubectl wait gatewayclass/cilium --for=condition=Accepted --timeout=3m -} - -@test "Cilium Gateway API controller reconciles a minimal Gateway to Programmed" { - # Use the pre-existing tenant-test namespace created by e2e-install-cozystack.bats. - kubectl apply -f - <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: gateway-e2e-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: http - protocol: HTTP - port: 80 - allowedRoutes: - namespaces: - from: Same -EOF - - # The controller must accept and program the Gateway. - kubectl -n tenant-test wait gateway/gateway-e2e-probe --for=condition=Accepted --timeout=2m - kubectl -n tenant-test wait gateway/gateway-e2e-probe --for=condition=Programmed --timeout=3m - - # Cilium materialises a LoadBalancer Service named cilium-gateway- - # for each programmed Gateway. Its existence is the observable proof that the - # full data-plane wiring kicked in. - kubectl -n tenant-test get svc cilium-gateway-gateway-e2e-probe - - # Cleanup - kubectl -n tenant-test delete gateway/gateway-e2e-probe --ignore-not-found --timeout=1m -} - -@test "cozystack-gateway-hostname-policy VAP and binding are installed" { - # Diagnose the hostname-policy enforcement path from the ground up. - # If these resources are missing, admission cannot reject anything. - kubectl get validatingadmissionpolicy cozystack-gateway-hostname-policy -o yaml - kubectl get validatingadmissionpolicybinding cozystack-gateway-hostname-policy -o yaml - # Binding MUST have validationActions: [Deny] — [Audit] / [] would let requests through silently. - actions=$(kubectl get validatingadmissionpolicybinding cozystack-gateway-hostname-policy -o jsonpath='{.spec.validationActions}') - echo "binding.validationActions=$actions" - case "$actions" in - *Deny*) ;; - *) echo "SETUP FAILURE: binding.validationActions lacks Deny (got '$actions')" >&2; return 1 ;; - esac -} - -@test "tenant-test namespace carries namespace.cozystack.io/host label from tenant chart" { - # Diagnostic: the whole hostname-policy VAP keys off this label. If it is - # missing or empty, the VAP's matchCondition returns false, VAP skips, and - # EVERY Gateway in the namespace is admitted regardless of listener hostname. - # Make that bug loud instead of letting it fall through as a silent pass. - host_label=$(kubectl get namespace tenant-test -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') - if [ -z "$host_label" ]; then - echo "SETUP FAILURE: tenant-test namespace lacks namespace.cozystack.io/host label" >&2 - kubectl get namespace tenant-test -o yaml >&2 - return 1 - fi - if [ "$host_label" != "test.example.org" ]; then - echo "SETUP FAILURE: tenant-test host label is '$host_label', expected 'test.example.org'" >&2 - kubectl get namespace tenant-test -o yaml >&2 - return 1 - fi -} - -@test "ValidatingAdmissionPolicy rejects Gateway with foreign hostname" { - # tenant-test namespace should only be allowed to publish its own - # domain suffix ('.test.example.org'); a listener hostname from the - # root tenant's apex must be denied by cozystack-gateway-hostname-policy. - # hack/cozytest.sh is /bin/sh (dash) with set -e — a failing command - # substitution propagates its exit status through variable assignment and - # kills the test. We specifically EXPECT admission to reject the apply, so - # use `if !` — set -e is disabled inside the if-condition, the exit status - # is captured, and $output is filled either way for the follow-up greps. - if ! output=$(kubectl apply -f - 2>&1 <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: hostname-hijack-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: https - protocol: HTTPS - port: 443 - hostname: "dashboard.example.org" - tls: - mode: Terminate - certificateRefs: - - name: noop - allowedRoutes: - namespaces: - from: Same -EOF -); then - # Happy path: admission rejected the Gateway. Verify the rejection came - # from our VAP and names the expected tenant host. - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "must equal test.example.org" - else - echo "BUG: admission accepted cross-tenant hostname — Gateway 'hostname-hijack-probe' was created in tenant-test" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "Package admission accepts gateway.attachedNamespaces with tenant-* entries" { - # Regression guard: the previous shape rejected `tenant-*` entries in - # publishing.gateway.attachedNamespaces (both a render-time fail in - # cozystack-basics and a dedicated VAP). Under inheritance the attach - # surface is governed by the namespace.cozystack.io/gateway label - # selector, not by entries in attachedNamespaces — Layers 4/5/7 - # (Tenant.spec.host, namespace label, HTTPRoute hostname VAPs) defend - # against hostname hijack independently. Adding a tenant-* entry must - # no longer be blocked at admission. A future refactor that re-adds - # the gate would silently break inheritance for every cluster that - # has set attachedNamespaces explicitly. - kubectl apply -f - <<'EOF' -apiVersion: cozystack.io/v1alpha1 -kind: Package -metadata: - name: vap-accept-probe -spec: - variant: isp-full - components: - platform: - values: - gateway: - attachedNamespaces: - - tenant-alice -EOF - # Apply succeeded → admission accepted the tenant-* entry. Clean up - # immediately; this Package was just a probe and is not consumed by - # the platform. - kubectl delete package vap-accept-probe --ignore-not-found -} - -@test "cozystack-tenant-host-policy blocks non-trusted callers from setting tenant.spec.host" { - # Impersonate a tenant-scoped ServiceAccount that is NOT in the trustedCaller - # group list. First grant RBAC to create Tenants — authorization runs BEFORE - # admission, so without this grant the apiserver returns a plain RBAC - # Forbidden and the test would fail grep-ing for 'ValidatingAdmissionPolicy' - # even though the VAP itself is fine. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: vap-probe-tenant-create - namespace: tenant-test -rules: -- apiGroups: ["apps.cozystack.io"] - resources: ["tenants"] - verbs: ["create","get","list","watch","update","patch","delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: vap-probe-tenant-create - namespace: tenant-test -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: vap-probe-tenant-create -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - apply -f - 2>&1 <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: vaphostprobe - namespace: tenant-test -spec: - host: foreign.example.org -EOF -); then - kubectl -n tenant-test delete rolebinding vap-probe-tenant-create --ignore-not-found - kubectl -n tenant-test delete role vap-probe-tenant-create --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "spec.host can only be set" - else - kubectl -n tenant-test delete tenants.apps.cozystack.io vaphostprobe --ignore-not-found - kubectl -n tenant-test delete rolebinding vap-probe-tenant-create --ignore-not-found - kubectl -n tenant-test delete role vap-probe-tenant-create --ignore-not-found - echo "BUG: admission accepted tenant.spec.host from untrusted SA — Tenant 'vaphostprobe' was created" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-namespace-host-label-policy blocks non-trusted callers from changing the host label" { - # tenant-test namespace already has namespace.cozystack.io/host set by the - # cozystack tenant chart. Grant patch on namespaces cluster-wide to the - # impersonated SA — namespaces is a cluster-scoped resource so this needs a - # ClusterRole. Authorization runs before admission, so without this grant - # the test would fail with plain RBAC Forbidden rather than a VAP rejection. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: vap-probe-namespace-patch -rules: -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get","list","watch","update","patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: vap-probe-namespace-patch -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: vap-probe-namespace-patch -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - label namespace tenant-test \ - namespace.cozystack.io/host=foreign.example.org --overwrite 2>&1); then - kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "immutable" - else - # Revert label if apiserver somehow accepted the overwrite. - kubectl label namespace tenant-test namespace.cozystack.io/host=test.example.org --overwrite - kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found - echo "BUG: admission accepted host label change from untrusted SA" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-namespace-host-label-policy blocks non-trusted callers from setting the host label at CREATE" { - # Defense-in-depth: a non-trusted caller must not be able to stamp - # namespace.cozystack.io/host=X on a brand-new namespace either — only - # cozystack/Flux SAs may write the label. Authorization runs before - # admission, so grant cluster-wide namespace create to the impersonated SA - # first, otherwise the test would fail with plain RBAC Forbidden. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: vap-probe-namespace-create -rules: -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["create","get","list","delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: vap-probe-namespace-create -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: vap-probe-namespace-create -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - apply -f - 2>&1 <<'EOF' -apiVersion: v1 -kind: Namespace -metadata: - name: vap-host-label-probe - labels: - namespace.cozystack.io/host: foreign.example.org -EOF -); then - kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "immutable" - else - kubectl delete namespace vap-host-label-probe --ignore-not-found --wait=false - kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found - echo "BUG: admission accepted first-time host label write from untrusted SA at CREATE — Namespace 'vap-host-label-probe' was created" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "TenantGateway CRD is installed (gateway.cozystack.io/v1alpha1)" { - # Shipped via packages/system/cozystack-controller/definitions; the - # controller cannot reconcile anything without it being Established. - kubectl wait crd/tenantgateways.gateway.cozystack.io --for=condition=Established --timeout=60s -} - -@test "TenantGatewayReconciler materialises Gateway + Issuer from a TenantGateway CR" { - # Create the CR directly (the chart renders one of these per tenant - # with tenant.spec.gateway=true; this test exercises the controller - # in isolation without going through the full tenant flow). - kubectl apply -f - <<'EOF' -apiVersion: gateway.cozystack.io/v1alpha1 -kind: TenantGateway -metadata: - name: tg-e2e-probe - namespace: tenant-test -spec: - apex: test.example.org - certMode: http01 - gatewayClassName: cilium -EOF - - # Controller creates the Gateway (same name, same namespace). - timeout 120 sh -ec 'until kubectl -n tenant-test get gateway tg-e2e-probe >/dev/null 2>&1; do sleep 2; done' - kubectl -n tenant-test get gateway tg-e2e-probe -o yaml - - # And the per-tenant ACME Issuer. - timeout 120 sh -ec 'until kubectl -n tenant-test get issuer.cert-manager.io tg-e2e-probe-gateway >/dev/null 2>&1; do sleep 2; done' - - # Status is reported back: ObservedGeneration tracks .metadata.generation, - # Ready=True after a clean reconcile. - timeout 120 sh -ec 'until kubectl -n tenant-test get tenantgateways.gateway.cozystack.io tg-e2e-probe -o jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}" 2>/dev/null | grep -q True; do sleep 2; done' - - # Cleanup. Cascade-delete relies on OwnerReferences set by the reconciler. - kubectl -n tenant-test delete tenantgateways.gateway.cozystack.io tg-e2e-probe --ignore-not-found --timeout=1m -} - -@test "cozystack-route-hostname-policy VAP rejects HTTPRoute claiming a foreign apex" { - # tenant-test's namespace.cozystack.io/host label is test.example.org. - # An HTTPRoute claiming attacker.com (or any hostname not under - # test.example.org) must be denied by Layer 7 of the security model. - if ! output=$(kubectl apply -f - 2>&1 <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: route-hostname-hijack-probe - namespace: tenant-test -spec: - hostnames: - - "attacker.com" - rules: - - backendRefs: - - name: kubernetes - namespace: default - port: 443 -EOF -); then - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -qi "hostnames must equal" - else - kubectl -n tenant-test delete httproute route-hostname-hijack-probe --ignore-not-found - echo "BUG: admission accepted cross-apex HTTPRoute hostname — route was created" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-route-hostname-policy VAP allows HTTPRoute under the namespace's apex" { - # Sanity check the inverse: a route with a hostname under - # test.example.org must NOT be rejected by the VAP. - kubectl apply -f - <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: route-hostname-allow-probe - namespace: tenant-test -spec: - hostnames: - - "harbor.test.example.org" - rules: - - backendRefs: - - name: kubernetes - namespace: default - port: 443 -EOF - - kubectl -n tenant-test delete httproute route-hostname-allow-probe --ignore-not-found -} - -@test "HTTPRoute with a matching parentRef reaches Accepted status" { - # Put a Gateway and a route in the same namespace so allowedRoutes: Same accepts them. - kubectl apply -f - <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: gateway-route-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: http - protocol: HTTP - port: 80 - allowedRoutes: - namespaces: - from: Same ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: httproute-probe - namespace: tenant-test -spec: - parentRefs: - - name: gateway-route-probe - sectionName: http - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - name: kubernetes - namespace: default - port: 443 -EOF - - kubectl -n tenant-test wait gateway/gateway-route-probe --for=condition=Programmed --timeout=3m - timeout 120 sh -ec 'until kubectl -n tenant-test get httproute httproute-probe -o jsonpath="{.status.parents[0].conditions[?(@.type==\"Accepted\")].status}" 2>/dev/null | grep -q True; do sleep 2; done' - - kubectl -n tenant-test delete httproute/httproute-probe --ignore-not-found - kubectl -n tenant-test delete gateway/gateway-route-probe --ignore-not-found -} - -@test "tenant chart writes namespace.cozystack.io/gateway label and _namespace.gateway in lockstep with tenant.gatewayEffective" { - # Pins the contract that namespace.yaml, gateway.yaml, and any other - # tenant-chart consumer of the gateway flag resolve through the same - # tenant.gatewayEffective helper. If they diverge — opt-in Tenant - # rendering Gateway HelmRelease but namespace.yaml leaving - # _namespace.gateway empty — child apps (harbor, bucket) silently - # fall back to Ingress while the Gateway resource sits unused. - # - # Uses explicit opt-in (tenant.spec.gateway=true) so the test is - # self-contained and does not require platform-level - # gateway.enabled=true in the e2e setup. The derived-apex - # auto-default path through _cluster.gateway-enabled is covered by - # helm-unittest fixtures in packages/apps/tenant/tests/ - # gateway_default_test.yaml; both paths converge in the same helper, - # so e2e only needs to exercise one to guard the lockstep contract. - # - # Provisions a child tenant under tenant-test with explicit - # gateway=true and asserts: - # - the child Namespace carries namespace.cozystack.io/gateway= - # - the cozystack-values Secret in the child namespace has - # _namespace.gateway: "" matching tenant.gatewayEffective - kubectl apply -f - <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: gwprop - namespace: tenant-test -spec: - gateway: true -EOF - timeout 180 sh -ec 'until kubectl get ns tenant-test-gwprop >/dev/null 2>&1; do sleep 3; done' - timeout 60 sh -ec 'until kubectl -n tenant-test-gwprop get secret cozystack-values >/dev/null 2>&1; do sleep 2; done' - - ns_label=$(kubectl get ns tenant-test-gwprop -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/gateway}') - [ "$ns_label" = "tenant-test-gwprop" ] - - cozyvalues=$(kubectl -n tenant-test-gwprop get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d) - echo "$cozyvalues" | grep -E '^\s*gateway:\s*"tenant-test-gwprop"\s*$' >/dev/null - - # Wait out the uninstall instead of fire-and-forget: tenant teardown - # runs a cleanup Job that the helm uninstall blocks on, and a dangling - # uninstall carries over into the next .bats file, starving the - # helm-controller worker pool (--concurrent=5 on the tenants shard). - kubectl -n tenant-test delete tenant gwprop --ignore-not-found - kubectl -n tenant-test wait hr tenant-gwprop --for=delete --timeout=300s -} - -@test "child tenant without explicit gateway inherits _namespace.gateway from a Gateway-owning parent" { - # Pins the inheritance flow: when a parent tenant owns its Gateway - # (gateway=true), a child tenant under that parent with the gateway - # field unset receives _namespace.gateway = in - # its cozystack-values Secret AND the same value on the - # namespace.cozystack.io/gateway label of the child namespace. - # - # Without this lockstep, the parent's Gateway label-selector - # allowedRoutes does not match the child namespace at runtime, and - # the child's HTTPRoutes silently fail to attach. - # - # tenant-test in the e2e fixture has no Gateway of its own, so the - # test sets up its own parent (gwparent) under tenant-test with - # explicit gateway=true. The child (gwchild) under that parent is - # the inheritance-under-test. - kubectl apply -f - <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: gwparent - namespace: tenant-test -spec: - gateway: true -EOF - timeout 180 sh -ec 'until kubectl get ns tenant-test-gwparent >/dev/null 2>&1; do sleep 3; done' - timeout 60 sh -ec 'until kubectl -n tenant-test-gwparent get secret cozystack-values >/dev/null 2>&1; do sleep 2; done' - parent_label=$(kubectl get ns tenant-test-gwparent -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/gateway}') - [ "$parent_label" = "tenant-test-gwparent" ] - - kubectl apply -f - <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: gwchild - namespace: tenant-test-gwparent -spec: {} -EOF - timeout 180 sh -ec 'until kubectl get ns tenant-test-gwparent-gwchild >/dev/null 2>&1; do sleep 3; done' - timeout 60 sh -ec 'until kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values >/dev/null 2>&1; do sleep 2; done' - - child_label=$(kubectl get ns tenant-test-gwparent-gwchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/gateway}') - [ "$child_label" = "tenant-test-gwparent" ] - - cozyvalues=$(kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d) - echo "$cozyvalues" | grep -E '^\s*gateway:\s*"tenant-test-gwparent"\s*$' >/dev/null - - # The child must NOT have its own gateway HelmRelease — inheritance - # means no separate Gateway resource for the child tenant. - ! kubectl -n tenant-test-gwparent-gwchild get helmrelease gateway 2>/dev/null - - # Teardown child before parent, waiting out each uninstall: deleting - # the parent while the child is still uninstalling wedges the parent's - # cleanup Job on the child's namespace, and both stuck uninstalls - # occupy helm-controller workers past the end of this .bats file. - kubectl -n tenant-test-gwparent delete tenant gwchild --ignore-not-found - kubectl -n tenant-test-gwparent wait hr tenant-gwchild --for=delete --timeout=300s - kubectl -n tenant-test delete tenant gwparent --ignore-not-found - kubectl -n tenant-test wait hr tenant-gwparent --for=delete --timeout=300s -} - -@test "child tenant's HTTPRoute drives the parent Gateway's listener set via inheritance label" { - # End-to-end cross-namespace attach for the inheritance flow: - # parent owns the Gateway, child inherits via the - # namespace.cozystack.io/gateway label, and an HTTPRoute in the - # child namespace causes the parent Gateway to grow a per-listener - # HTTPS entry for the route's hostname plus a per-listener - # Certificate object. Both writes are cozystack-controller's - # responsibility — they happen only if collectHostnameClaims sees - # the route across the inheritance label. - # - # The Cilium-side `HTTPRoute.status.parents[].conditions[Accepted]` - # flip is NOT checked here. In the e2e cluster the ACME server - # (LE prod) refuses to issue any certificate for `.example.org` - # (forbidden by policy), so a fresh per-listener cert never goes - # Ready, which Cilium ties to listener readiness, which blocks - # Accepted on the route. Every cluster cert visible in cozyreport - # (alerta, grafana, dashboard, seaweedfs-s3) shows the same - # `rejectedIdentifier` for the same reason — bootstrap-time certs - # survive on stale issuance, fresh ones can't be issued. Asserting - # Accepted=True against that environment is environmental noise, - # not a contract on the inheritance code path. - # - # The route's hostname is constructed under the child's derived - # apex (.) so Layer 7 - # (cozystack-route-hostname-policy) admits it. - kubectl apply -f - <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: rparent - namespace: tenant-test -spec: - gateway: true -EOF - timeout 180 sh -ec 'until kubectl get ns tenant-test-rparent >/dev/null 2>&1; do sleep 3; done' - # Wait for the cozystack-controller to materialise the parent's - # Gateway resource. Programmed status requires Cilium dataplane - # provisioning which depends on the LB allocator; for this test - # we only need the Gateway object to exist + carry the - # label-selector allowedRoutes. - timeout 120 sh -ec 'until kubectl -n tenant-test-rparent get gateway cozystack >/dev/null 2>&1; do sleep 3; done' - - parent_apex=$(kubectl get ns tenant-test-rparent -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') - [ -n "$parent_apex" ] - - kubectl apply -f - </dev/null 2>&1; do sleep 3; done' - - # Verify the inheritance label propagated to the child namespace — - # this is the read side of the contract collectHostnameClaims - # exercises. - child_gateway_label=$(kubectl get ns tenant-test-rparent-rchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/gateway}') - [ "$child_gateway_label" = "tenant-test-rparent" ] - - child_apex=$(kubectl get ns tenant-test-rparent-rchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') - [ -n "$child_apex" ] - - route_host="harbor.${child_apex}" - - kubectl apply -f - </dev/null \ - | grep -qx "$want"; do - sleep 3 - done - ' - - # reconcilePerListenerCertificates runs from the same dynHostnames - # slice — a missing Certificate here would mean the controller - # rendered the listener but not its cert ref, which would silently - # leak listeners with broken TLS refs. - timeout 60 sh -ec ' - want="'"$route_host"'" - until kubectl -n tenant-test-rparent get certificate \ - -l cozystack.io/per-listener-cert=true \ - -o jsonpath="{range .items[*]}{.spec.dnsNames[0]}{\"\n\"}{end}" 2>/dev/null \ - | grep -qx "$want"; do - sleep 3 - done - ' - - kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found - # Teardown child before parent, waiting out each uninstall (see the - # inheritance test above). The rchild HR may still be mid-install when - # the delete lands — helm-controller finishes the install action before - # it can uninstall, so the child wait also absorbs that tail instead of - # leaving it for the next .bats file. - kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found - kubectl -n tenant-test-rparent wait hr tenant-rchild --for=delete --timeout=300s - kubectl -n tenant-test delete tenant rparent --ignore-not-found - kubectl -n tenant-test wait hr tenant-rparent --for=delete --timeout=300s -} diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats deleted file mode 100644 index 8fe300939..000000000 --- a/hack/e2e-apps/harbor.bats +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bats - -@test "Create Harbor" { - name='test' - release="harbor-$name" - - # Clean up stale resources from previous failed runs - kubectl -n tenant-test delete harbor.apps.cozystack.io $name 2>/dev/null || true - kubectl -n tenant-test wait hr $release --timeout=60s --for=delete 2>/dev/null || true - - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr $release --timeout=5m --for=condition=ready - - # Wait for COSI to provision the bucket. The driver creates the backend Bucket - # and grants access; the central COSI controller requeues the BucketClaim until - # the Bucket's readiness has propagated, so it converges within tens of seconds. - # A frozen claim (the propagation race this guards against) never converges, so - # the tight bound surfaces that regression instead of masking it. - timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \ - --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { - echo "=== BucketClaim did not converge to bucketReady=true ===" - kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true - echo "=== backend Buckets (cluster-scoped) ===" - kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true - echo "=== objectstorage-controller ===" - kubectl -n cozy-objectstorage-controller get pods 2>&1 || true - false - } - timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \ - --timeout=60s --for=jsonpath='{.status.accessGranted}'=true - - kubectl -n tenant-test wait hr $release-system --timeout=600s --for=condition=ready || { - echo "=== HelmRelease status ===" - kubectl -n tenant-test get hr $release-system -o yaml 2>&1 || true - echo "=== Pods ===" - kubectl -n tenant-test get pods 2>&1 || true - echo "=== Events ===" - kubectl -n tenant-test get events --sort-by='.lastTimestamp' 2>&1 | tail -30 || true - echo "=== ExternalArtifact ===" - kubectl -n cozy-system get externalartifact cozystack-harbor-application-default-harbor-system -o yaml 2>&1 || true - echo "=== BucketClaim status ===" - kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true - echo "=== BucketAccess status ===" - kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true - echo "=== BucketAccess Secret ===" - kubectl -n tenant-test get secret $release-registry-bucket -o jsonpath='{.data.BucketInfo}' 2>&1 | base64 -d 2>&1 || true - false - } - timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-core >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait deploy $release-core --timeout=120s --for=condition=available - timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-registry >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait deploy $release-registry --timeout=120s --for=condition=available - timeout 60 sh -ec "until kubectl -n tenant-test get deploy $release-portal >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait deploy $release-portal --timeout=120s --for=condition=available - kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.admin-password}' | base64 --decode | grep -q '.' - kubectl -n tenant-test get secret $release-credentials -o jsonpath='{.data.url}' | base64 --decode | grep -q 'https://' - kubectl -n tenant-test get svc $release -o jsonpath='{.spec.ports[0].port}' | grep -q '80' - kubectl -n tenant-test delete harbor.apps.cozystack.io $name -} diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats deleted file mode 100644 index d0a8266b9..000000000 --- a/hack/e2e-apps/kafka.bats +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bats - -@test "Create Kafka" { - name='test' - kubectl -n tenant-test delete kafka.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl -n tenant-test wait kafka.apps.cozystack.io $name --for=delete --timeout=2m 2>/dev/null || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr kafka-$name --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get kafkas $name >/dev/null 2>&1; do sleep 2; done" - kubectl wait kafkas -n tenant-test $name --timeout=300s --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done" - kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound - timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-1 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-1 --timeout=50s --for=jsonpath='{.status.phase}'=Bound - timeout 40 sh -ec "until kubectl -n tenant-test get svc kafka-$name-zookeeper-client -o jsonpath='{.spec.ports[0].port}' | grep -q '2181'; do sleep 10; done" - timeout 40 sh -ec "until kubectl -n tenant-test get svc kafka-$name-zookeeper-nodes -o jsonpath='{.spec.ports[*].port}' | grep -q '2181 2888 3888'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints kafka-$name-zookeeper-nodes -o jsonpath='{.subsets[*].addresses[0].ip}' | grep -q '[0-9]'; do sleep 10; done" - kubectl -n tenant-test delete kafka.apps.cozystack.io $name --timeout=2m - # Strimzi reclaims ZooKeeper PVCs automatically when deleteClaim is true on - # the persistent-claim storage. Verify they disappear instead of issuing a - # manual kubectl delete, which would mask a regression in the operator. - timeout 60 sh -ec "while kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0 >/dev/null 2>&1; do sleep 2; done" - timeout 60 sh -ec "while kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-1 >/dev/null 2>&1; do sleep 2; done" -} diff --git a/hack/e2e-apps/kubernetes-latest.bats b/hack/e2e-apps/kubernetes-latest.bats deleted file mode 100644 index 3674f83c9..000000000 --- a/hack/e2e-apps/kubernetes-latest.bats +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bats - -# Sourced at file scope (not inside the @test) so cozytest.sh sources it into -# the parent shell: both run_kubernetes_test and the cozy_cleanup() hook the -# EXIT trap calls must live there (the @test itself runs in a subshell). -. hack/e2e-apps/run-kubernetes.sh - -@test "Create a tenant Kubernetes control plane with latest version" { - # 4th arg = enable_ouroboros: folds the standalone ouroboros.bats's - # hairpin-NAT reconciliation assertions onto this cluster instead of - # spinning up a second ~25m Kamaji bringup. - run_kubernetes_test 'keys | sort_by(.) | .[-1]' 'test-latest-version' '59991' 'true' -} diff --git a/hack/e2e-apps/kubernetes-previous.bats b/hack/e2e-apps/kubernetes-previous.bats deleted file mode 100644 index ac84bd9a0..000000000 --- a/hack/e2e-apps/kubernetes-previous.bats +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bats - -# Sourced at file scope (see kubernetes-latest.bats) so cozytest.sh's parent -# shell gets run_kubernetes_test + the cozy_cleanup() hook for the EXIT trap. -. hack/e2e-apps/run-kubernetes.sh - -@test "Create a tenant Kubernetes control plane with previous version" { - run_kubernetes_test 'keys | sort_by(.) | .[-2]' 'test-previous-version' '59992' -} diff --git a/hack/e2e-apps/kuberture.bats b/hack/e2e-apps/kuberture.bats deleted file mode 100644 index 04e2aaaec..000000000 --- a/hack/e2e-apps/kuberture.bats +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bats - -# Smoke-test the kuberture system addon: a Package CR for cozystack.kuberture -# with config.outputs configured must materialise an HR in cozy-kuberture, -# the controller Deployment must roll out, and the controller must create the -# configured headless Services with the right external-dns annotations -# (hostname/target/ttl) reflecting the IPs from default/kubernetes -# EndpointSlice. Two outputs with different annotationPrefix values exercise -# the cozystack-relevant property: a single kuberture instance can address -# multiple external-dns instances from one platform-level package. -# -# cozytest.sh is plain POSIX shell — no `[[ ... ]]`, no bats `run` helper, -# no FD 3, and `teardown()` is never auto-invoked. Cleanup is inlined at the -# end of the @test body and runs on the success path only; the e2e Makefile's -# sandbox teardown handles the failure case. - -# Shared cleanup so the inline success-path call at the end of the @test -# body and the teardown hook (alive only under upstream bats) cannot -# drift apart. -_cleanup() { - # Probe Pods and their RBAC live outside the chart and need explicit - # teardown — the Package delete cascade does not own them. The - # ClusterRole/ClusterRoleBinding are cluster-scoped and would otherwise - # leak across e2e re-runs on the same sandbox cluster. - kubectl delete --namespace cozy-kuberture --ignore-not-found pod/kuberture-e2e-edns-public pod/kuberture-e2e-edns-internal 2>/dev/null || true - kubectl delete --namespace cozy-kuberture --ignore-not-found sa/kuberture-e2e-edns-probe 2>/dev/null || true - kubectl delete --ignore-not-found clusterrolebinding/kuberture-e2e-edns-probe clusterrole/kuberture-e2e-edns-probe 2>/dev/null || true - kubectl delete package.cozystack.io cozystack.kuberture --ignore-not-found --wait=true --timeout=180s 2>/dev/null || true -} - -teardown() { - # Dead code under cozytest.sh — the runner never invokes it. Kept so - # the file still works under upstream bats (e.g. for local debugging - # via `bats hack/e2e-apps/kuberture.bats`). - _cleanup -} - -@test "Platform-level kuberture publishes default/kubernetes EndpointSlice as annotated headless Services for multiple external-dns instances" { - pub_host=kuberture-e2e-public.cozystack.test - int_host=kuberture-e2e-internal.cozystack.test - pub_prefix=external-dns.alpha.kubernetes.io/ - int_prefix=kuberture-e2e-internal-dns.io/ - - kubectl apply --filename - </dev/null 2>&1; do sleep 2; done' - timeout 180 sh -ec 'until kubectl --namespace cozy-kuberture get hr kuberture >/dev/null 2>&1; do sleep 2; done' - kubectl --namespace cozy-kuberture wait hr kuberture --timeout=300s --for=condition=ready - - # Controller rollout. Use kubectl wait so the readiness check is anchored on - # the Available condition and not on a substring match of the replica count. - kubectl --namespace cozy-kuberture wait deployment kuberture --for=condition=available --timeout=180s - - # The output Services are created reactively after the controller observes - # default/kubernetes EndpointSlice. Wait for both to appear. - for svc in kuberture-e2e-public kuberture-e2e-internal; do - timeout 120 sh -ec "until kubectl --namespace cozy-kuberture get service ${svc} >/dev/null 2>&1; do sleep 3; done" - done - - # Both Services must be headless and carry external-dns annotations under - # their configured prefix. The `target` annotation must be non-empty — - # the actual IPs come from default/kubernetes EndpointSlice and depend on - # the cluster, so we only assert presence. - pub_json=$(kubectl --namespace cozy-kuberture get service kuberture-e2e-public --output json) - int_json=$(kubectl --namespace cozy-kuberture get service kuberture-e2e-internal --output json) - - test "$(echo "${pub_json}" | jq --raw-output '.spec.clusterIP')" = "None" - test "$(echo "${int_json}" | jq --raw-output '.spec.clusterIP')" = "None" - - test "$(echo "${pub_json}" | jq --raw-output --arg k "${pub_prefix}hostname" '.metadata.annotations[$k]')" = "${pub_host}" - test "$(echo "${int_json}" | jq --raw-output --arg k "${int_prefix}hostname" '.metadata.annotations[$k]')" = "${int_host}" - - pub_target=$(echo "${pub_json}" | jq --raw-output --arg k "${pub_prefix}target" '.metadata.annotations[$k] // ""') - int_target=$(echo "${int_json}" | jq --raw-output --arg k "${int_prefix}target" '.metadata.annotations[$k] // ""') - test -n "${pub_target}" - test -n "${int_target}" - - test "$(echo "${pub_json}" | jq --raw-output --arg k "${pub_prefix}ttl" '.metadata.annotations[$k]')" = "60" - test "$(echo "${int_json}" | jq --raw-output --arg k "${int_prefix}ttl" '.metadata.annotations[$k]')" = "300" - - # The cross-prefix assertion proves the per-output isolation: the public - # Service must NOT carry the internal prefix's annotation, and vice versa. - test "$(echo "${pub_json}" | jq --raw-output --arg k "${int_prefix}hostname" '.metadata.annotations[$k] // "absent"')" = "absent" - test "$(echo "${int_json}" | jq --raw-output --arg k "${pub_prefix}hostname" '.metadata.annotations[$k] // "absent"')" = "absent" - - # Routing proof: run two short-lived external-dns probes, each with a - # distinct --annotation-prefix, and verify each one picks up exactly its - # matching Service. This is the upstream Split Horizon DNS pattern from - # kubernetes-sigs/external-dns docs/advanced/split-horizon. inmemory - # provider + noop registry + --once means each probe runs one sync cycle - # against the live Services in cozy-kuberture, logs what it would publish, - # and exits — the logs are the assertion surface. - # external-dns 0.20 needs cluster-wide read on services/endpoints/nodes/ - # namespaces even when `--namespace` narrows the source scope (informer - # cache initialisation reads cluster-wide before applying the filter). - # A namespace-scoped Role surfaces as a Pod crash without an obvious - # error in the bats log, so use ClusterRole + ClusterRoleBinding here. - kubectl apply --filename - </dev/null | grep -E '^(Succeeded|Failed)$' >/dev/null; do sleep 3; done" - phase=$(kubectl --namespace cozy-kuberture get pod kuberture-e2e-edns-${probe} -o jsonpath='{.status.phase}') - if [ "${phase}" != "Succeeded" ]; then - # Dump enough context to diagnose the failure from the bats log alone - # (the e2e sandbox is destroyed before the run artefacts ship, so - # post-mortem kubectl access on the cluster is not available). - echo "==== probe ${probe} failed: phase=${phase}; describe ====" - kubectl --namespace cozy-kuberture describe pod kuberture-e2e-edns-${probe} || true - echo "==== probe ${probe} logs (current container) ====" - kubectl --namespace cozy-kuberture logs pod/kuberture-e2e-edns-${probe} || true - echo "==== probe ${probe} logs (previous container, if any) ====" - kubectl --namespace cozy-kuberture logs pod/kuberture-e2e-edns-${probe} --previous || true - exit 1 - fi - done - - pub_logs=$(kubectl --namespace cozy-kuberture logs pod/kuberture-e2e-edns-public) - int_logs=$(kubectl --namespace cozy-kuberture logs pod/kuberture-e2e-edns-internal) - - # The probe that reads the default external-dns prefix must see - # kuberture-e2e-public.cozystack.test (kuberture-e2e-public Service carries - # external-dns.alpha.kubernetes.io/hostname=...) and must NOT see the - # internal hostname (kuberture-e2e-internal carries only the internal prefix). - echo "${pub_logs}" | grep -F "${pub_host}" >/dev/null - ! echo "${pub_logs}" | grep -F "${int_host}" >/dev/null - - # And the inverse for the internal-prefix probe. - echo "${int_logs}" | grep -F "${int_host}" >/dev/null - ! echo "${int_logs}" | grep -F "${pub_host}" >/dev/null - - # _cleanup deletes the probe Pods, their SA, the ClusterRole+ClusterRoleBinding, - # and finally the Package CR. Inlined here for the success path so a re-run - # within the same e2e session starts clean; failure path is covered by the - # sandbox teardown. - _cleanup -} diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats deleted file mode 100644 index 0c1e3a5e8..000000000 --- a/hack/e2e-apps/mariadb.bats +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bats - -@test "Create DB MariaDB" { - name='test' - kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready - timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2 - timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" - timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 - kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name -} diff --git a/hack/e2e-apps/mongodb.bats b/hack/e2e-apps/mongodb.bats deleted file mode 100644 index 0750e4317..000000000 --- a/hack/e2e-apps/mongodb.bats +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bats - -@test "Create DB MongoDB" { - name='test' - kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - # Wait for HelmRelease - kubectl -n tenant-test wait hr mongodb-$name --timeout=5m --for=condition=ready - # Wait for MongoDB service (port 27017) - timeout 120 sh -ec "until kubectl -n tenant-test get svc mongodb-$name-rs0 -o jsonpath='{.spec.ports[0].port}' | grep -q '27017'; do sleep 10; done" - # Wait for endpoints - timeout 180 sh -ec "until kubectl -n tenant-test get endpoints mongodb-$name-rs0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - # Wait for StatefulSet replicas - timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mongodb-$name-rs0 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait statefulset.apps/mongodb-$name-rs0 --timeout=300s --for=jsonpath='{.status.replicas}'=1 - # Cleanup - kubectl -n tenant-test delete mongodbs.apps.cozystack.io $name -} diff --git a/hack/e2e-apps/openbao.bats b/hack/e2e-apps/openbao.bats deleted file mode 100644 index 81ca42b3e..000000000 --- a/hack/e2e-apps/openbao.bats +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bats - -@test "Create OpenBAO (standalone)" { - name='test' - kubectl -n tenant-test delete openbao.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr openbao-$name --timeout=5m --for=condition=ready - kubectl -n tenant-test wait hr openbao-$name-system --timeout=5m --for=condition=ready - - # Wait for container to be started (pod Running does not guarantee container is ready for exec on slow CI) - if ! timeout 120 sh -ec "until kubectl -n tenant-test get pod openbao-$name-0 --output jsonpath='{.status.containerStatuses[0].started}' 2>/dev/null | grep -q true; do sleep 5; done"; then - echo "=== DEBUG: Container did not start in time ===" >&2 - kubectl -n tenant-test describe pod openbao-$name-0 >&2 || true - kubectl -n tenant-test logs openbao-$name-0 --previous >&2 || true - kubectl -n tenant-test logs openbao-$name-0 >&2 || true - return 1 - fi - - # Wait for OpenBAO API to accept connections - # bao status exit codes: 0 = unsealed, 1 = error/not ready, 2 = sealed but responsive - if ! timeout 60 sh -ec "until kubectl -n tenant-test exec openbao-$name-0 -- bao status >/dev/null 2>&1; rc=\$?; test \$rc -eq 0 -o \$rc -eq 2; do sleep 3; done"; then - echo "=== DEBUG: OpenBAO API did not become responsive ===" >&2 - kubectl -n tenant-test describe pod openbao-$name-0 >&2 || true - kubectl -n tenant-test logs openbao-$name-0 --previous >&2 || true - kubectl -n tenant-test logs openbao-$name-0 >&2 || true - return 1 - fi - - # Initialize OpenBAO (single key share for testing simplicity) - init_output=$(kubectl -n tenant-test exec openbao-$name-0 -- bao operator init -key-shares=1 -key-threshold=1 -format=json) - unseal_key=$(echo "$init_output" | jq -r '.unseal_keys_b64[0]') - if [ -z "$unseal_key" ] || [ "$unseal_key" = "null" ]; then - echo "Failed to extract unseal key. Init output: $init_output" >&2 - return 1 - fi - - # Unseal OpenBAO - kubectl -n tenant-test exec openbao-$name-0 -- bao operator unseal "$unseal_key" - - # Now wait for pod to become ready (readiness probe checks seal status) - timeout 60 sh -ec "until kubectl -n tenant-test get sts openbao-$name >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait sts openbao-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 - timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-openbao-$name-0 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait pvc data-openbao-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound - kubectl -n tenant-test delete openbao.apps.cozystack.io $name - kubectl -n tenant-test delete pvc data-openbao-$name-0 --ignore-not-found -} diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats deleted file mode 100644 index 44c0c5bd7..000000000 --- a/hack/e2e-apps/postgres.bats +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bats - -@test "Create DB PostgreSQL" { - name='test' - kubectl -n tenant-test delete postgreses.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr postgres-$name --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete - timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" - timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" - timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-rw -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done" - timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-r -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - # for some reason it takes longer for the read-only endpoint to be ready - #timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-ro -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-rw -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" - kubectl -n tenant-test delete postgreses.apps.cozystack.io $name - kubectl -n tenant-test delete job.batch/postgres-$name-init-job -} diff --git a/hack/e2e-apps/qdrant.bats b/hack/e2e-apps/qdrant.bats deleted file mode 100755 index 7d031a1d7..000000000 --- a/hack/e2e-apps/qdrant.bats +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bats - -@test "Create Qdrant" { - name='test' - kubectl -n tenant-test delete qdrant.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr qdrant-$name --timeout=5m --for=condition=ready - kubectl -n tenant-test wait hr qdrant-$name-system --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get sts qdrant-$name >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait sts qdrant-$name --timeout=90s --for=jsonpath='{.status.readyReplicas}'=1 - timeout 60 sh -ec "until kubectl -n tenant-test get pvc qdrant-storage-qdrant-$name-0 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait pvc qdrant-storage-qdrant-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound - kubectl -n tenant-test delete qdrant.apps.cozystack.io $name -} diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats deleted file mode 100644 index 98eb4595e..000000000 --- a/hack/e2e-apps/redis.bats +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bats - -@test "Create Redis" { - name='test' - kubectl -n tenant-test delete redis.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl apply -f- </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr redis-$name --timeout=5m --for=condition=ready - timeout 60 sh -ec "until kubectl -n tenant-test get pvc redisfailover-persistent-data-rfr-redis-$name-0 >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound - timeout 60 sh -ec "until kubectl -n tenant-test get deploy rfs-redis-$name >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available - timeout 60 sh -ec "until kubectl -n tenant-test get sts rfr-redis-$name >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2 - kubectl -n tenant-test delete redis.apps.cozystack.io $name -} diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats deleted file mode 100644 index 79b0ef6b9..000000000 --- a/hack/e2e-apps/vminstance.bats +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bats - -@test "Create a VM Disk" { - name='test' - # Delete any leftover from a previous run and BLOCK until removal completes. - # `kubectl apply` on a resource still in finalizer-drain silently no-ops - # ("Detected changes to resource ... which is currently being deleted"), - # which then races a NotFound on the downstream HR wait. The previous - # `|| true` swallowed timeout errors here and let the test continue with - # the old VMDisk still draining. Bumped timeout to 3m and removed `|| true` - # so a true delete failure surfaces immediately. - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr vm-disk-$name --timeout=5m --for=condition=ready - kubectl -n tenant-test wait dv vm-disk-$name --timeout=250s --for=condition=ready - kubectl -n tenant-test wait pvc vm-disk-$name --timeout=200s --for=jsonpath='{.status.phase}'=Bound - # Drop the VMDisk so the next test starts from a clean slate. Each test - # owns its own resources; "Create a VM Instance" creates its own VMDisk - # inline rather than depending on this one. - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $name --ignore-not-found --timeout=3m -} - -@test "Create a VM Instance" { - # Use a distinct disk name from "Create a VM Disk" so a flaky finalizer - # drain on the prior disk's PV/PVC cannot collide with this test's apply. - diskName='vmi-test' - name='test' - # Pre-clean any leftover VMInstance from a prior run. The VMDisk is - # created by this test (decoupled from "Create a VM Disk" above), so no - # predelete here — the prior test already cleaned up its own VMDisk. - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m - # Create the VMDisk this test depends on. Same wait-for-HR pattern as the - # first test, so this block is self-contained. - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait hr vm-disk-$diskName --timeout=5m --for=condition=ready - kubectl -n tenant-test wait dv vm-disk-$diskName --timeout=250s --for=condition=ready - kubectl -n tenant-test wait pvc vm-disk-$diskName --timeout=200s --for=jsonpath='{.status.phase}'=Bound - kubectl apply -f - </dev/null 2>&1; do sleep 2; done" - # Wait for the parent HR Ready before polling for the VMI. Under Flux v2.8 - # kstatus the helm install can be still dispatching when the previous - # ordering assumed it had already applied the VirtualMachine CR — without - # this the next poll for `vmi vm-instance-$name` returned NotFound for - # its entire 120s budget because kubevirt hadn't seen the VM yet. - kubectl -n tenant-test wait hr vm-instance-$name --timeout=5m --for=condition=ready - # Nested KubeVirt VM startup (virt-launcher + libvirt + cloud-init DHCP) - # routinely takes 30-60s under runner load. 5m is a comfortable upper - # bound for nested virt + slow runner I/O after the HR-Ready gate. - timeout 5m sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 2; done" - # VM ready follows IP assignment closely; 60s gives buffer for the qemu-guest-agent. - kubectl -n tenant-test wait vm vm-instance-$name --timeout=5m --for=condition=ready - kubectl -n tenant-test delete vminstances.apps.cozystack.io $name --ignore-not-found --timeout=3m - kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName --ignore-not-found --timeout=3m -} diff --git a/hack/e2e-chainsaw/.chainsaw.yaml b/hack/e2e-chainsaw/.chainsaw.yaml index f019b765b..c518e7647 100644 --- a/hack/e2e-chainsaw/.chainsaw.yaml +++ b/hack/e2e-chainsaw/.chainsaw.yaml @@ -16,8 +16,11 @@ spec: timeouts: apply: 1m assert: 5m - delete: 2m - cleanup: 2m + # delete/cleanup are generous: finalizer-draining CRs (VMDisk, FoundationDB, + # Tenant child namespaces) can take minutes to fully reclaim their backing + # resources, and Chainsaw waits for the delete to complete during cleanup. + delete: 5m + cleanup: 5m error: 5m exec: 5m cleanup: diff --git a/hack/e2e-chainsaw/.gitignore b/hack/e2e-chainsaw/.gitignore new file mode 100644 index 000000000..45e36c625 --- /dev/null +++ b/hack/e2e-chainsaw/.gitignore @@ -0,0 +1,4 @@ +# Chainsaw JUnit report artifacts (written into the run directory; CI uploads +# them via docker cp, they are never committed). +*.xml +chainsaw-report*.xml diff --git a/hack/e2e-chainsaw/README.md b/hack/e2e-chainsaw/README.md index f6d7f1518..d86674c1c 100644 --- a/hack/e2e-chainsaw/README.md +++ b/hack/e2e-chainsaw/README.md @@ -1,95 +1,75 @@ -# Chainsaw E2E tests (pilot) +# Chainsaw E2E tests -This directory is a pilot for migrating the BATS E2E app tests (`hack/e2e-apps/*.bats`) to [Kyverno Chainsaw](https://github.com/kyverno/chainsaw) — a declarative, Kubernetes-native E2E test framework (CNCF, part of the Kyverno project; successor to KUTTL). +The Cozystack E2E app suite, written for [Kyverno Chainsaw](https://github.com/kyverno/chainsaw) — a declarative, Kubernetes-native E2E test framework (CNCF, part of the Kyverno project; successor to KUTTL). -Two representative tests are ported as examples: +This directory replaces the per-app BATS suite that used to live in `hack/e2e-apps/*.bats`. The CI `e2e` job runs `chainsaw test hack/e2e-chainsaw/` after `make install-cozystack`; cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. -| Chainsaw test | BATS original | -|---|---| -| [`postgres/`](./postgres/) | `hack/e2e-apps/postgres.bats` | -| [`bucket/`](./bucket/) | `hack/e2e-apps/bucket.bats` | +## Layout -## Why - -The BATS suite carries ~100 copies of the same polling boilerplate: - -```bash -timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done" -kubectl -n tenant-test wait hr postgres-$name --timeout=5m --for=condition=ready +``` +hack/e2e-chainsaw/ +├── .chainsaw.yaml # global Configuration (namespace, timeouts, JUnit report) +├── _lib/ # shell reused by the script-heavy suites +│ ├── run-kubernetes.sh # tenant Kubernetes bringup + LB/NFS/ouroboros checks +│ └── remediation-guard.sh +├── / +│ ├── chainsaw-test.yaml # one or more Test definitions +│ └── .yaml # the manifests the test applies +└── ... ``` -The two-step dance exists because `kubectl wait` fails immediately when the object does not exist yet. In Chainsaw the same logic is one declarative assertion that polls existence and state together: +Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `qdrant`, `clickhouse`, `kafka`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `kubernetes-latest`, `kubernetes-previous`. -```yaml -- assert: - timeout: 6m - resource: - apiVersion: helm.toolkit.fluxcd.io/v2 - kind: HelmRelease - metadata: - name: postgres-test - status: - (conditions[?type == 'Ready']): - - status: "True" -``` +## What Chainsaw buys over the BATS suite -What this buys us: - -- **Readable tests** — a test is a list of resources and expected states, not shell control flow. No `pipefail` traps, no `|| true`, no `grep` on jsonpath. -- **Distinguishable failure modes** — "object never created" and "created but never ready" both surface as an assertion timeout with a structured diff of expected vs. actual state; "wrong status" shows the exact mismatched field. Negative tests (a state that *must* be reached, e.g. a webhook rejection) use the `error` operation. -- **Configurable timeouts** at four levels: global config → test → step → operation. -- **Automatic diagnostics** — `catch` blocks dump events, `describe` output, and pod logs on any failure. Today only `harbor.bats` does this, by hand. -- **Automatic cleanup** — resources applied by Chainsaw are deleted (and waited on) in reverse order when the test finishes, pass or fail. -- **Custom checks** — `script` operations cover anything imperative (`psql` probe, `mc` S3 access checks), with JMESPath checks on stdout/stderr/exit code. -- **CI-ready output** — JUnit XML report (`chainsaw-report.xml`), built-in parallelism for when tests stop sharing fixed ports/names. +- **Readable tests** — a list of resources and expected states, not shell control flow. The `timeout N sh -ec "until kubectl get ..."` + `kubectl wait` pair that appeared ~100 times collapses into a single `assert` that polls existence and state together. +- **Distinguishable failure modes** — "object never created", "created but never ready", and "wrong status" all surface as a structured diff instead of an opaque shell timeout. +- **Automatic diagnostics** — `catch` blocks dump events, `describe`, and pod logs on any failure (the BATS suite only did this in `harbor.bats`, by hand). +- **Automatic cleanup** — resources Chainsaw applied are deleted (and waited on) in reverse order; `finally` covers operator-created leftovers. +- **CI-ready output** — a JUnit `chainsaw-report.xml` is emitted for the test summary. ## Test anatomy -Each test is a directory with a `chainsaw-test.yaml` (the `Test` definition) plus the manifests it applies: +Each suite is a directory with a `chainsaw-test.yaml` (one or more `Test` docs) plus the manifests it applies. A step has `try` (operations: `apply`, `assert`, `error`, `script`, `delete`, …), optional `catch` (diagnostics on failure), and optional `finally` (teardown regardless of outcome). -``` -postgres/ -├── chainsaw-test.yaml # steps: apply → assert ready → probe; catch; finally -└── postgres.yaml # the Postgres CR, namespace injected by chainsaw -``` +Declarative suites assert on `status.conditions` and concrete fields. Inherently imperative suites keep their logic in `script` steps: -A step has three blocks: +- **openbao** — `bao operator init`/`unseal` via `kubectl exec`. +- **kuberture** — `jq` on Service annotations + external-dns split-horizon log assertions. +- **gateway** — admission-rejection cases that need `kubectl --as` impersonation or error-message greps; the tenant apex is read from the namespace `namespace.cozystack.io/host` label at runtime so they pass on any host. +- **kubernetes-latest/previous** — wrap `_lib/run-kubernetes.sh` (Kamaji bringup, port-forward, LB/NFS/ouroboros). The script `cd`s to the repo root (the test dir is `hack/e2e-chainsaw/`, so repo root is `../../..`). -- `try` — the operations (`apply`, `assert`, `error`, `script`, `delete`, …) -- `catch` — diagnostics on failure (`events`, `describe`, `podLogs`, …); also definable test-wide in `spec.catch` -- `finally` — teardown that runs regardless of outcome +## Conventions / gotchas (Chainsaw v0.2.15) + +- Condition asserts use the **filter-as-list** form, not the documented `[0]` index: + ```yaml + status: + (conditions[?type == 'Ready']): + - status: "True" + ``` + The `(conditions[?type == 'Ready'])[0]: {status: "True"}` idiom fails with "field not found in the input object". +- A reading of a failure diff: the **"actual" side is a projection onto the expected fields** — `status: {}` means the JMESPath expression failed, not that status is empty. Read the error line above the diff first. +- `error` with a bare resource ref (name only) asserts the resource **does not exist** (used for "PVC reclaimed", "no backup schedule", "child has no Gateway HR"). +- `script` runs in the **test directory** with `$NAMESPACE` set to the config namespace (`tenant-test`). ## Running locally -Requires a cluster with Cozystack installed and the `tenant-test` namespace (the same environment the BATS suite runs against, see `hack/e2e-install-cozystack.bats`). The bucket test additionally needs `mc`, `jq`, and `nc` on the host, same as its BATS original. +Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` additionally needs `mc` and `nc` on the host; `kuberture`/`openbao`/`harbor` need `jq`. ```bash # install chainsaw -go install github.com/kyverno/chainsaw@latest -# or: brew install kyverno/chainsaw/chainsaw +go install github.com/kyverno/chainsaw@latest # or: brew install kyverno/chainsaw/chainsaw -# run all tests (config is picked up from .chainsaw.yaml) +# run all suites (config picked up from .chainsaw.yaml) cd hack/e2e-chainsaw chainsaw test -# run a single test +# run a single suite chainsaw test postgres/ ``` -## CI integration (follow-up, not in this PR) +`kubernetes-latest`/`kubernetes-previous` each bring up a full tenant Kubernetes cluster (~25 min, real worker nodes) and `vminstance` boots a nested VM — expect a long run for the full set. -The suite slots into the existing sandbox flow next to `cozytest.sh`: run `chainsaw test hack/e2e-chainsaw/` after `make install-cozystack`, upload `chainsaw-report.xml` as an artifact. Once parity is proven, each migrated `.bats` file is deleted; the per-test 3-retry loop in `.github/workflows/pull-requests.yaml` can likely be reduced, since assertion polling removes the fixed-timeout flakiness most retries paper over. +## CI integration -## BATS → Chainsaw cheat sheet - -| BATS construction | Chainsaw equivalent | -|---|---| -| `kubectl apply -f - < and +# -system), not with an app-type prefix. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: external-dns-inmemory +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: test-system + - podLogs: + selector: app.kubernetes.io/instance=test-system + tail: 50 + steps: + - name: create-externaldns + try: + - apply: + file: externaldns-inmemory.yaml + - name: wait-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: test-system + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: test-system + status: + readyReplicas: 1 +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: external-dns-prefix +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: test-prefix-system + - podLogs: + selector: app.kubernetes.io/instance=test-prefix-system + tail: 50 + steps: + - name: create-externaldns + try: + - apply: + file: externaldns-prefix.yaml + - name: wait-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: test-prefix + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: test-prefix-system + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: test-prefix-system + status: + readyReplicas: 1 diff --git a/hack/e2e-chainsaw/external-dns/externaldns-inmemory.yaml b/hack/e2e-chainsaw/external-dns/externaldns-inmemory.yaml new file mode 100644 index 000000000..7ab4e3752 --- /dev/null +++ b/hack/e2e-chainsaw/external-dns/externaldns-inmemory.yaml @@ -0,0 +1,8 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: ExternalDNS +metadata: + name: test +spec: + provider: inmemory + domainFilters: + - example.com diff --git a/hack/e2e-chainsaw/external-dns/externaldns-prefix.yaml b/hack/e2e-chainsaw/external-dns/externaldns-prefix.yaml new file mode 100644 index 000000000..f86699cc8 --- /dev/null +++ b/hack/e2e-chainsaw/external-dns/externaldns-prefix.yaml @@ -0,0 +1,9 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: ExternalDNS +metadata: + name: test-prefix +spec: + provider: inmemory + annotationPrefix: custom-dns/ + domainFilters: + - example.org diff --git a/hack/e2e-chainsaw/foundationdb/chainsaw-test.yaml b/hack/e2e-chainsaw/foundationdb/chainsaw-test.yaml new file mode 100644 index 000000000..f3a1cfb52 --- /dev/null +++ b/hack/e2e-chainsaw/foundationdb/chainsaw-test.yaml @@ -0,0 +1,166 @@ +# Chainsaw port of hack/e2e-apps/foundationdb.bats ("Create DB FoundationDB"). +# +# The dense block of `until kubectl get ... jsonpath | grep` status probes +# collapses into one assert on the FoundationDBCluster status tree. Pod-count +# (by process class + phase) and PVC-size checks stay as polling scripts — +# they count List results, which an assert cannot express. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: foundationdb +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: foundationdb-test + - describe: + apiVersion: apps.foundationdb.org/v1beta2 + kind: FoundationDBCluster + name: foundationdb-test + - podLogs: + selector: foundationdb.org/fdb-cluster-name=foundationdb-test + tail: 50 + steps: + - name: create-foundationdb + try: + - apply: + file: foundationdb.yaml + + - name: wait-helmrelease-ready + try: + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: foundationdb-test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: assert-cluster-status + try: + - assert: + timeout: 5m + resource: + apiVersion: apps.foundationdb.org/v1beta2 + kind: FoundationDBCluster + metadata: + name: foundationdb-test + status: + configured: true + hasListenIPsForAllPods: true + databaseConfiguration: + logs: 3 + proxies: 3 + redundancy_mode: double + resolvers: 1 + storage_engine: ssd-2 + usable_regions: 1 + health: + available: true + fullReplication: true + healthy: true + # volumeClaimTemplate size is pinned on the CR spec. + - assert: + resource: + apiVersion: apps.foundationdb.org/v1beta2 + kind: FoundationDBCluster + metadata: + name: foundationdb-test + spec: + processes: + general: + volumeClaimTemplate: + spec: + resources: + requests: + storage: 1Gi + + - name: verify-processes-and-security-context + try: + - script: + timeout: 5m + content: | + set -eu + ns="$NAMESPACE" + cl=foundationdb-test + sel="foundationdb.org/fdb-cluster-name=$cl" + count_running() { + kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=$1" \ + --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l + } + deadline=$(( $(date +%s) + 240 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + s=$(count_running storage); l=$(count_running log); c=$(count_running cluster_controller) + echo "storage=$s log=$l cluster_controller=$c" + if [ "$s" -eq 3 ] && [ "$l" -ge 1 ] && [ "$c" -eq 1 ]; then break; fi + sleep 10 + done + test "$(count_running storage)" -eq 3 + test "$(count_running log)" -ge 1 + test "$(count_running cluster_controller)" -eq 1 + # securityContext: storage pod must run as the non-root 4059 uid/gid. + pod=$(kubectl -n "$ns" get pods -l "$sel,foundationdb.org/fdb-process-class=storage" \ + --no-headers | head -n1 | awk '{print $1}') + kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsUser}' | grep -qx 4059 + kubectl -n "$ns" get pod "$pod" -o jsonpath='{.spec.containers[0].securityContext.runAsGroup}' | grep -qx 4059 + # Status fields that are nil early on (so they live here, not in the + # assert, where contains()/numeric-compare on nil hard-errors). + fdb() { kubectl -n "$ns" get foundationdbclusters.apps.foundationdb.org "$cl" -o jsonpath="$1"; } + deadline=$(( $(date +%s) + 120 )) + until fdb '{.status.connectionString}' | grep -qE '@.*\.svc\.cozy\.local'; do + [ "$(date +%s)" -ge "$deadline" ] && { echo "connectionString not set" >&2; fdb '{.status.connectionString}' >&2; exit 1; } + sleep 5 + done + fdb '{.status.desiredProcessGroups}' | grep -qE '^[0-9]+$' + fdb '{.status.generations.reconciled}' | grep -qE '^[0-9]+$' + check: + ($error == null): true + + - name: verify-monitoring-and-storage + try: + # Dashboard resource map ConfigMap is created by the controller. + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: foundationdb-test-resourcemap + - script: + timeout: 2m + content: | + set -eu + ns="$NAMESPACE" + cl=foundationdb-test + # WorkloadMonitor is created and configured with replicas=3. + kubectl -n "$ns" get workloadmonitor "$cl" + kubectl -n "$ns" get workloadmonitor "$cl" -o jsonpath='{.spec.replicas}' | grep -qx 3 + # PVCs: at least 3, all requesting and reporting 1Gi. + count=$(kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" --no-headers | wc -l) + test "$count" -ge 3 + kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \ + -o jsonpath='{.items[*].spec.resources.requests.storage}' | grep -q '1Gi' + kubectl -n "$ns" get pvc -l "foundationdb.org/fdb-cluster-name=$cl" \ + -o jsonpath='{.items[*].status.capacity.storage}' | grep -q '1Gi' + check: + ($error == null): true + + - name: verify-cleanup + try: + - delete: + ref: + apiVersion: apps.cozystack.io/v1alpha1 + kind: FoundationDB + name: test + - error: + timeout: 2m + resource: + apiVersion: apps.foundationdb.org/v1beta2 + kind: FoundationDBCluster + metadata: + name: foundationdb-test diff --git a/hack/e2e-chainsaw/foundationdb/foundationdb.yaml b/hack/e2e-chainsaw/foundationdb/foundationdb.yaml new file mode 100644 index 000000000..78061a6df --- /dev/null +++ b/hack/e2e-chainsaw/foundationdb/foundationdb.yaml @@ -0,0 +1,36 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: FoundationDB +metadata: + name: test +spec: + cluster: + version: "7.3.63" + processCounts: + storage: 3 + stateless: -1 + cluster_controller: 1 + redundancyMode: "double" + storageEngine: "ssd-2" + faultDomain: + key: "foundationdb.org/none" + valueFrom: "$FDB_ZONE_ID" + storage: + size: "1Gi" + storageClass: "" + resourcesPreset: "small" + backup: + enabled: false + s3: + bucket: "" + endpoint: "" + region: "" + credentials: + accessKeyId: "" + secretAccessKey: "" + retentionPolicy: "7d" + monitoring: + enabled: true + customParameters: + - "knob_disable_posix_kernel_aio=1" + imageType: "unified" + automaticReplacements: true diff --git a/hack/e2e-chainsaw/gateway/chainsaw-test.yaml b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml new file mode 100644 index 000000000..a73d9a86f --- /dev/null +++ b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml @@ -0,0 +1,759 @@ +# Chainsaw port of hack/e2e-apps/gateway.bats (16 cases). +# +# Declarative cases (CRD/GatewayClass/Gateway/HTTPRoute/TenantGateway status, +# VAP existence) use asserts. Admission-rejection cases need kubectl +# impersonation or error-message greps that Chainsaw `apply` can't express, so +# they stay scripts. The tenant apex is read at runtime from the tenant-test +# namespace's namespace.cozystack.io/host label, so these tests pass both in CI +# (test.example.org) and on a dev cluster with a different host. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gateway-api-crds +spec: + steps: + - name: crds-established-and-gatewayclass-accepted + try: + - assert: + timeout: 1m + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: gatewayclasses.gateway.networking.k8s.io + status: + (conditions[?type == 'Established']): + - status: "True" + - assert: + timeout: 1m + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: gateways.gateway.networking.k8s.io + status: + (conditions[?type == 'Established']): + - status: "True" + - assert: + timeout: 1m + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: httproutes.gateway.networking.k8s.io + status: + (conditions[?type == 'Established']): + - status: "True" + - assert: + timeout: 3m + resource: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: cilium + status: + (conditions[?type == 'Accepted']): + - status: "True" +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gateway-minimal-programmed +spec: + catch: + - events: {} + - describe: + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + name: gateway-e2e-probe + steps: + - name: create-minimal-gateway + try: + - apply: + file: gw-minimal.yaml + - name: assert-accepted-programmed + try: + - assert: + timeout: 2m + resource: + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-e2e-probe + status: + (conditions[?type == 'Accepted']): + - status: "True" + (conditions[?type == 'Programmed']): + - status: "True" + # Cilium materialises a LoadBalancer Service per programmed Gateway. + - assert: + timeout: 1m + resource: + apiVersion: v1 + kind: Service + metadata: + name: cilium-gateway-gateway-e2e-probe +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gateway-hostname-policy-installed +spec: + steps: + - name: vap-and-binding-and-host-label + try: + - assert: + resource: + apiVersion: admissionregistration.k8s.io/v1 + kind: ValidatingAdmissionPolicy + metadata: + name: cozystack-gateway-hostname-policy + - assert: + resource: + apiVersion: admissionregistration.k8s.io/v1 + kind: ValidatingAdmissionPolicyBinding + metadata: + name: cozystack-gateway-hostname-policy + # Binding must Deny — [Audit]/[] would let requests through silently. + (contains(spec.validationActions, 'Deny')): true + # The whole hostname policy keys off this label; assert it is set + # (script, not an assert: numeric/length compares on labels are unreliable). + - script: + content: | + set -eu + kubectl get namespace "$NAMESPACE" \ + -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}' | grep -q . + check: + ($error == null): true +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gateway-foreign-hostname-rejected +spec: + steps: + - name: foreign-listener-hostname-denied + try: + - script: + content: | + set -eu + out=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: hostname-hijack-probe + spec: + gatewayClassName: cilium + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: "dashboard.example.org" + tls: + mode: Terminate + certificateRefs: + - name: noop + allowedRoutes: + namespaces: + from: Same + EOF + ) && rc=0 || rc=$? + echo "rc=$rc out=$out" + if [ "$rc" -eq 0 ]; then + kubectl -n "$NAMESPACE" delete gateway hostname-hijack-probe --ignore-not-found + echo "BUG: admission accepted cross-tenant hostname" >&2 + exit 1 + fi + echo "$out" | grep -qi "ValidatingAdmissionPolicy" + echo "$out" | grep -q "must equal" + check: + ($error == null): true +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: package-attachednamespaces-accept +spec: + steps: + # Admission must accept a tenant-* entry in gateway.attachedNamespaces; a + # successful apply (and Chainsaw auto-cleanup of the Package) is the proof. + - name: apply-package-probe + try: + - apply: + file: package-attach-probe.yaml +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: tenant-host-policy +spec: + steps: + - name: untrusted-sa-cannot-set-tenant-host + try: + - script: + content: | + set -eu + ns="$NAMESPACE" + # Authorization runs before admission; grant the SA RBAC to create + # Tenants so the apply reaches the VAP rather than failing on RBAC. + kubectl apply -f - <&1 </dev/null 2>&1 || true + kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found + kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found + } + trap cleanup EXIT + out=$(kubectl --as="system:serviceaccount:${ns}:default" \ + --as-group=system:serviceaccounts \ + --as-group="system:serviceaccounts:${ns}" \ + label namespace "$ns" namespace.cozystack.io/host=foreign.example.org --overwrite 2>&1) && rc=0 || rc=$? + echo "rc=$rc out=$out" + [ "$rc" -ne 0 ] + echo "$out" | grep -qi "ValidatingAdmissionPolicy" + echo "$out" | grep -q "immutable" + check: + ($error == null): true + - name: untrusted-sa-cannot-set-host-label-at-create + try: + - script: + content: | + set -eu + ns="$NAMESPACE" + kubectl apply -f - <&1 <&1 <<'EOF' + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: route-hostname-hijack-probe + spec: + hostnames: + - "attacker.com" + rules: + - backendRefs: + - name: kubernetes + namespace: default + port: 443 + EOF + ) && rc=0 || rc=$? + echo "rc=$rc out=$out" + if [ "$rc" -eq 0 ]; then + kubectl -n "$NAMESPACE" delete httproute route-hostname-hijack-probe --ignore-not-found + echo "BUG: admission accepted cross-apex HTTPRoute hostname" >&2 + exit 1 + fi + echo "$out" | grep -qi "ValidatingAdmissionPolicy" + echo "$out" | grep -qi "hostnames must equal" + check: + ($error == null): true + - name: own-apex-route-allowed + try: + - script: + content: | + set -eu + ns="$NAMESPACE" + apex=$(kubectl get namespace "$ns" -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') + kubectl apply -n "$ns" -f - </dev/null 2>&1; do + [ "$(date +%s)" -ge "$deadline" ] && { echo "cozystack-values not created" >&2; exit 1; } + sleep 2 + done + v=$(kubectl -n tenant-test-gwprop get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d) + echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwprop"\s*$' + check: + ($error == null): true + finally: + - delete: + ref: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + name: gwprop +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: tenant-gateway-inheritance +spec: + catch: + - events: {} + steps: + - name: create-parent-tenant + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: gwparent + spec: + gateway: true + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-gwparent + labels: + namespace.cozystack.io/gateway: tenant-test-gwparent + - name: create-child-tenant + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: gwchild + namespace: tenant-test-gwparent + spec: {} + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-gwparent-gwchild + labels: + namespace.cozystack.io/gateway: tenant-test-gwparent + - name: assert-inheritance-values + try: + - script: + timeout: 2m + content: | + set -eu + deadline=$(( $(date +%s) + 60 )) + until kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values >/dev/null 2>&1; do + [ "$(date +%s)" -ge "$deadline" ] && { echo "child cozystack-values not created" >&2; exit 1; } + sleep 2 + done + v=$(kubectl -n tenant-test-gwparent-gwchild get secret cozystack-values -o jsonpath='{.data.values\.yaml}' | base64 -d) + echo "$v" | grep -E '^\s*gateway:\s*"tenant-test-gwparent"\s*$' + check: + ($error == null): true + # The child must NOT own a gateway HelmRelease — inheritance, not its own. + - error: + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: gateway + namespace: tenant-test-gwparent-gwchild + finally: + - delete: + ref: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + name: gwchild + namespace: tenant-test-gwparent + - delete: + ref: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + name: gwparent +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: tenant-gateway-route-listener +spec: + catch: + - events: {} + steps: + - name: parent-child-and-route-drive-listener-set + try: + - script: + timeout: 6m + content: | + set -eu + ns="$NAMESPACE" + + cleanup() { + kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true + kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null 2>&1 || true + kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null 2>&1 || true + } + trap cleanup EXIT + + # Parent tenant owns the Gateway. + kubectl apply -n "$ns" -f - </dev/null 2>&1; do sleep 3; done' + timeout 120 sh -ec 'until kubectl -n tenant-test-rparent get gateway cozystack >/dev/null 2>&1; do sleep 3; done' + + # Child tenant inherits the parent Gateway. + kubectl apply -n tenant-test-rparent -f - </dev/null 2>&1; do sleep 3; done' + + child_apex=$(kubectl get ns tenant-test-rparent-rchild -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') + [ -n "$child_apex" ] + route_host="harbor.${child_apex}" + + # An HTTPRoute in the child, attached to the parent Gateway. + kubectl apply -n tenant-test-rparent-rchild -f - </dev/null \ + | grep -qx "'"$route_host"'"; do sleep 3; done' + + # ...and a matching per-listener Certificate. + timeout 60 sh -ec ' + until kubectl -n tenant-test-rparent get certificate \ + -l cozystack.io/per-listener-cert=true \ + -o jsonpath="{range .items[*]}{.spec.dnsNames[0]}{\"\n\"}{end}" 2>/dev/null \ + | grep -qx "'"$route_host"'"; do sleep 3; done' + check: + ($error == null): true diff --git a/hack/e2e-chainsaw/gateway/gw-minimal.yaml b/hack/e2e-chainsaw/gateway/gw-minimal.yaml new file mode 100644 index 000000000..4e89cc0a1 --- /dev/null +++ b/hack/e2e-chainsaw/gateway/gw-minimal.yaml @@ -0,0 +1,13 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway-e2e-probe +spec: + gatewayClassName: cilium + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same diff --git a/hack/e2e-chainsaw/gateway/gw-route-probe.yaml b/hack/e2e-chainsaw/gateway/gw-route-probe.yaml new file mode 100644 index 000000000..d56aebccf --- /dev/null +++ b/hack/e2e-chainsaw/gateway/gw-route-probe.yaml @@ -0,0 +1,31 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway-route-probe +spec: + gatewayClassName: cilium + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-probe +spec: + parentRefs: + - name: gateway-route-probe + sectionName: http + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: kubernetes + namespace: default + port: 443 diff --git a/hack/e2e-chainsaw/gateway/package-attach-probe.yaml b/hack/e2e-chainsaw/gateway/package-attach-probe.yaml new file mode 100644 index 000000000..e08ab86ec --- /dev/null +++ b/hack/e2e-chainsaw/gateway/package-attach-probe.yaml @@ -0,0 +1,12 @@ +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: vap-accept-probe +spec: + variant: isp-full + components: + platform: + values: + gateway: + attachedNamespaces: + - tenant-alice diff --git a/hack/e2e-chainsaw/harbor/chainsaw-test.yaml b/hack/e2e-chainsaw/harbor/chainsaw-test.yaml new file mode 100644 index 000000000..2c9fd8281 --- /dev/null +++ b/hack/e2e-chainsaw/harbor/chainsaw-test.yaml @@ -0,0 +1,135 @@ +# Chainsaw port of hack/e2e-apps/harbor.bats ("Create Harbor"). +# +# The BATS test hand-rolled a large diagnostic dump on the harbor-test-system +# HelmRelease wait; here that becomes the test-wide catch block, which fires on +# any step failure. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: harbor +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: harbor-test + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: harbor-test-system + - describe: + apiVersion: objectstorage.k8s.io/v1alpha1 + kind: BucketClaim + name: harbor-test-registry + - describe: + apiVersion: objectstorage.k8s.io/v1alpha1 + kind: BucketAccess + name: harbor-test-registry + - podLogs: + selector: app.kubernetes.io/instance=harbor-test + tail: 50 + steps: + - name: create-harbor + try: + - apply: + file: harbor.yaml + + - name: wait-helmrelease-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: harbor-test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: wait-cosi-bucket + try: + - assert: + timeout: 5m + resource: + apiVersion: objectstorage.k8s.io/v1alpha1 + kind: BucketClaim + metadata: + name: harbor-test-registry + status: + bucketReady: true + - assert: + timeout: 1m + resource: + apiVersion: objectstorage.k8s.io/v1alpha1 + kind: BucketAccess + metadata: + name: harbor-test-registry + status: + accessGranted: true + + - name: wait-system-and-deployments + try: + - assert: + timeout: 10m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: harbor-test-system + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: harbor-test-core + status: + (conditions[?type == 'Available']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: harbor-test-registry + status: + (conditions[?type == 'Available']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: harbor-test-portal + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: verify-credentials-and-service + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: harbor-test + spec: + (ports[0].port): 80 + - script: + content: | + set -eu + pw=$(kubectl -n "$NAMESPACE" get secret harbor-test-credentials \ + -o jsonpath='{.data.admin-password}' | base64 -d) + url=$(kubectl -n "$NAMESPACE" get secret harbor-test-credentials \ + -o jsonpath='{.data.url}' | base64 -d) + test -n "$pw" + echo "$url" | grep -q 'https://' + check: + ($error == null): true diff --git a/hack/e2e-chainsaw/harbor/harbor.yaml b/hack/e2e-chainsaw/harbor/harbor.yaml new file mode 100644 index 000000000..71e2fe0a1 --- /dev/null +++ b/hack/e2e-chainsaw/harbor/harbor.yaml @@ -0,0 +1,27 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Harbor +metadata: + name: test +spec: + host: "" + storageClass: "" + core: + resources: {} + resourcesPreset: "nano" + registry: + resources: {} + resourcesPreset: "nano" + jobservice: + resources: {} + resourcesPreset: "nano" + trivy: + enabled: false + size: 2Gi + resources: {} + resourcesPreset: "nano" + database: + size: 2Gi + replicas: 1 + redis: + size: 1Gi + replicas: 1 diff --git a/hack/e2e-chainsaw/kafka/chainsaw-test.yaml b/hack/e2e-chainsaw/kafka/chainsaw-test.yaml new file mode 100644 index 000000000..ee1852e5b --- /dev/null +++ b/hack/e2e-chainsaw/kafka/chainsaw-test.yaml @@ -0,0 +1,115 @@ +# Chainsaw port of hack/e2e-apps/kafka.bats ("Create Kafka"). +# +# The final step replaces the BATS check that Strimzi reclaims the ZooKeeper +# PVCs on delete: instead of a manual kubectl delete, an explicit delete of the +# Kafka CR followed by `error` ops asserts both PVCs disappear (a regression in +# the operator's deleteClaim handling would leave them behind). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: kafka +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: kafka-test + - podLogs: + selector: app.kubernetes.io/instance=kafka-test + tail: 50 + steps: + - name: create-kafka + try: + - apply: + file: kafka.yaml + + - name: wait-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: kafka-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Kafka + metadata: + name: test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: verify-zookeeper + try: + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-kafka-test-zookeeper-0 + status: + phase: Bound + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-kafka-test-zookeeper-1 + status: + phase: Bound + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: kafka-test-zookeeper-client + spec: + (ports[0].port): 2181 + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: kafka-test-zookeeper-nodes + spec: + (ports[*].port): [2181, 2888, 3888] + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: kafka-test-zookeeper-nodes + (subsets[0].addresses[0].ip != null): true + + - name: verify-pvc-reclaim-on-delete + try: + - delete: + ref: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Kafka + name: test + - error: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-kafka-test-zookeeper-0 + - error: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-kafka-test-zookeeper-1 diff --git a/hack/e2e-chainsaw/kafka/kafka.yaml b/hack/e2e-chainsaw/kafka/kafka.yaml new file mode 100644 index 000000000..e9138da6d --- /dev/null +++ b/hack/e2e-chainsaw/kafka/kafka.yaml @@ -0,0 +1,32 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Kafka +metadata: + name: test +spec: + external: false + kafka: + size: 10Gi + replicas: 2 + storageClass: "" + resources: {} + resourcesPreset: "nano" + zookeeper: + size: 5Gi + replicas: 2 + storageClass: "" + resources: {} + resourcesPreset: "nano" + topics: + - name: testResults + partitions: 1 + replicas: 2 + config: + min.insync.replicas: 2 + - name: testOrders + config: + cleanup.policy: compact + segment.ms: 3600000 + max.compaction.lag.ms: 5400000 + min.insync.replicas: 2 + partitions: 1 + replicas: 2 diff --git a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml new file mode 100644 index 000000000..72e75f019 --- /dev/null +++ b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml @@ -0,0 +1,31 @@ +# Chainsaw port of hack/e2e-apps/kubernetes-latest.bats. +# +# Spinning up a tenant Kubernetes control plane (Kamaji) + worker nodes + the +# LB/NFS/ouroboros checks is deeply imperative (port-forwards, a tenant +# kubeconfig, curl probes), so the battle-tuned logic in +# _lib/run-kubernetes.sh is reused verbatim from a single script step rather +# than rewritten as asserts. The script cd's to the repo root (the test dir is +# hack/e2e-chainsaw/kubernetes-latest, so repo root is ../../..) where the +# helper's `helm template` and versions.yaml paths resolve. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: kubernetes-latest +spec: + catch: + - events: {} + steps: + - name: tenant-kubernetes-latest-version + try: + - script: + # ~25m Kamaji bringup + worker join + LB/NFS/ouroboros assertions. + timeout: 40m + content: | + set -eu + cd ../../.. + . hack/e2e-chainsaw/_lib/run-kubernetes.sh + # 4th arg "true" folds the ouroboros hairpin-NAT assertions onto this + # cluster instead of a second ~25m bringup. + run_kubernetes_test 'keys | sort_by(.) | .[-1]' 'test-latest-version' '59991' 'true' + check: + ($error == null): true diff --git a/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml new file mode 100644 index 000000000..3e2eee6d3 --- /dev/null +++ b/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml @@ -0,0 +1,23 @@ +# Chainsaw port of hack/e2e-apps/kubernetes-previous.bats. See the sibling +# kubernetes-latest test for why the logic stays a script wrapping +# _lib/run-kubernetes.sh. This one targets the previous K8s version and skips +# the ouroboros addon (no 4th arg). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: kubernetes-previous +spec: + catch: + - events: {} + steps: + - name: tenant-kubernetes-previous-version + try: + - script: + timeout: 40m + content: | + set -eu + cd ../../.. + . hack/e2e-chainsaw/_lib/run-kubernetes.sh + run_kubernetes_test 'keys | sort_by(.) | .[-2]' 'test-previous-version' '59992' + check: + ($error == null): true diff --git a/hack/e2e-chainsaw/kuberture/chainsaw-test.yaml b/hack/e2e-chainsaw/kuberture/chainsaw-test.yaml new file mode 100644 index 000000000..43b891801 --- /dev/null +++ b/hack/e2e-chainsaw/kuberture/chainsaw-test.yaml @@ -0,0 +1,151 @@ +# Chainsaw port of hack/e2e-apps/kuberture.bats. The kuberture addon publishes +# the default/kubernetes EndpointSlice as annotated headless Services for +# multiple external-dns instances. Resource readiness is declarative; the +# annotation-shape checks and the split-horizon log assertions stay scripts. +# +# Everything is applied by chainsaw (the cluster-scoped Package, the cluster +# RBAC, the probe Pods), so cleanup is automatic — no inlined teardown. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: kuberture +spec: + catch: + - events: + namespace: cozy-kuberture + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: kuberture + namespace: cozy-kuberture + - podLogs: + namespace: cozy-kuberture + selector: app.kubernetes.io/name=kuberture + tail: 100 + steps: + - name: create-package + try: + - apply: + file: package.yaml + + - name: wait-kuberture-ready + try: + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: cozy-kuberture + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: kuberture + namespace: cozy-kuberture + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 3m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kuberture + namespace: cozy-kuberture + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: wait-output-services + try: + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Service + metadata: + name: kuberture-e2e-public + namespace: cozy-kuberture + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Service + metadata: + name: kuberture-e2e-internal + namespace: cozy-kuberture + + - name: assert-annotation-shape + try: + - script: + content: | + set -eu + pub_host=kuberture-e2e-public.cozystack.test + int_host=kuberture-e2e-internal.cozystack.test + pub_prefix=external-dns.alpha.kubernetes.io/ + int_prefix=kuberture-e2e-internal-dns.io/ + + pub_json=$(kubectl -n cozy-kuberture get service kuberture-e2e-public -o json) + int_json=$(kubectl -n cozy-kuberture get service kuberture-e2e-internal -o json) + + # Both Services must be headless. + test "$(echo "$pub_json" | jq -r '.spec.clusterIP')" = "None" + test "$(echo "$int_json" | jq -r '.spec.clusterIP')" = "None" + + # hostname/ttl carry the configured per-output values. + test "$(echo "$pub_json" | jq -r --arg k "${pub_prefix}hostname" '.metadata.annotations[$k]')" = "$pub_host" + test "$(echo "$int_json" | jq -r --arg k "${int_prefix}hostname" '.metadata.annotations[$k]')" = "$int_host" + test "$(echo "$pub_json" | jq -r --arg k "${pub_prefix}ttl" '.metadata.annotations[$k]')" = "60" + test "$(echo "$int_json" | jq -r --arg k "${int_prefix}ttl" '.metadata.annotations[$k]')" = "300" + + # target is populated from the EndpointSlice (value is cluster-specific). + test -n "$(echo "$pub_json" | jq -r --arg k "${pub_prefix}target" '.metadata.annotations[$k] // ""')" + test -n "$(echo "$int_json" | jq -r --arg k "${int_prefix}target" '.metadata.annotations[$k] // ""')" + + # Per-output isolation: neither Service carries the other prefix. + test "$(echo "$pub_json" | jq -r --arg k "${int_prefix}hostname" '.metadata.annotations[$k] // "absent"')" = "absent" + test "$(echo "$int_json" | jq -r --arg k "${pub_prefix}hostname" '.metadata.annotations[$k] // "absent"')" = "absent" + check: + ($error == null): true + + - name: split-horizon-routing-proof + try: + - apply: + file: probes.yaml + - script: + timeout: 4m + content: | + set -eu + for probe in public internal; do + deadline=$(( $(date +%s) + 180 )) + until kubectl -n cozy-kuberture get pod "kuberture-e2e-edns-${probe}" \ + -o jsonpath='{.status.phase}' 2>/dev/null | grep -Eq '^(Succeeded|Failed)$'; do + [ "$(date +%s)" -ge "$deadline" ] && break + sleep 3 + done + phase=$(kubectl -n cozy-kuberture get pod "kuberture-e2e-edns-${probe}" -o jsonpath='{.status.phase}') + if [ "$phase" != "Succeeded" ]; then + echo "==== probe ${probe} failed: phase=${phase} ====" + kubectl -n cozy-kuberture describe pod "kuberture-e2e-edns-${probe}" || true + kubectl -n cozy-kuberture logs "pod/kuberture-e2e-edns-${probe}" || true + exit 1 + fi + done + + pub_host=kuberture-e2e-public.cozystack.test + int_host=kuberture-e2e-internal.cozystack.test + pub_logs=$(kubectl -n cozy-kuberture logs pod/kuberture-e2e-edns-public) + int_logs=$(kubectl -n cozy-kuberture logs pod/kuberture-e2e-edns-internal) + + # The default-prefix probe sees only the public hostname; the internal- + # prefix probe sees only the internal hostname. + echo "$pub_logs" | grep -Fq "$pub_host" + ! echo "$pub_logs" | grep -Fq "$int_host" + echo "$int_logs" | grep -Fq "$int_host" + ! echo "$int_logs" | grep -Fq "$pub_host" + check: + ($error == null): true diff --git a/hack/e2e-chainsaw/kuberture/package.yaml b/hack/e2e-chainsaw/kuberture/package.yaml new file mode 100644 index 000000000..719e725d9 --- /dev/null +++ b/hack/e2e-chainsaw/kuberture/package.yaml @@ -0,0 +1,26 @@ +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.kuberture +spec: + variant: default + components: + kuberture: + values: + kuberture: + config: + outputs: + - name: public + hostname: + - kuberture-e2e-public.cozystack.test + annotationPrefix: external-dns.alpha.kubernetes.io/ + serviceName: kuberture-e2e-public + addressSource: endpointslice + recordTTL: 60 + - name: internal + hostname: + - kuberture-e2e-internal.cozystack.test + annotationPrefix: kuberture-e2e-internal-dns.io/ + serviceName: kuberture-e2e-internal + addressSource: endpointslice + recordTTL: 300 diff --git a/hack/e2e-chainsaw/kuberture/probes.yaml b/hack/e2e-chainsaw/kuberture/probes.yaml new file mode 100644 index 000000000..692e5b182 --- /dev/null +++ b/hack/e2e-chainsaw/kuberture/probes.yaml @@ -0,0 +1,82 @@ +# external-dns probe SA/RBAC + two single-shot probe Pods, each with a distinct +# --annotation-prefix (the upstream Split Horizon DNS pattern). external-dns +# 0.20 initialises its informer cache cluster-wide before applying --namespace, +# so the probe needs a ClusterRole, not a namespaced Role. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kuberture-e2e-edns-probe + namespace: cozy-kuberture +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kuberture-e2e-edns-probe +rules: + - apiGroups: [""] + resources: ["services", "endpoints", "pods", "nodes", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kuberture-e2e-edns-probe +subjects: + - kind: ServiceAccount + name: kuberture-e2e-edns-probe + namespace: cozy-kuberture +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kuberture-e2e-edns-probe +--- +apiVersion: v1 +kind: Pod +metadata: + name: kuberture-e2e-edns-public + namespace: cozy-kuberture +spec: + serviceAccountName: kuberture-e2e-edns-probe + restartPolicy: Never + containers: + - name: edns + image: registry.k8s.io/external-dns/external-dns:v0.20.0 + args: + - --source=service + - --namespace=cozy-kuberture + - --provider=inmemory + - --inmemory-zone=cozystack.test + - --registry=noop + - --once + - --interval=10s + - --annotation-prefix=external-dns.alpha.kubernetes.io/ + - --domain-filter=cozystack.test + - --log-level=info + - --log-format=text +--- +apiVersion: v1 +kind: Pod +metadata: + name: kuberture-e2e-edns-internal + namespace: cozy-kuberture +spec: + serviceAccountName: kuberture-e2e-edns-probe + restartPolicy: Never + containers: + - name: edns + image: registry.k8s.io/external-dns/external-dns:v0.20.0 + args: + - --source=service + - --namespace=cozy-kuberture + - --provider=inmemory + - --inmemory-zone=cozystack.test + - --registry=noop + - --once + - --interval=10s + - --annotation-prefix=kuberture-e2e-internal-dns.io/ + - --domain-filter=cozystack.test + - --log-level=info + - --log-format=text diff --git a/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml b/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml new file mode 100644 index 000000000..a00a54451 --- /dev/null +++ b/hack/e2e-chainsaw/mariadb/chainsaw-test.yaml @@ -0,0 +1,89 @@ +# Chainsaw port of hack/e2e-apps/mariadb.bats ("Create DB MariaDB"). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: mariadb +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: mariadb-test + - podLogs: + selector: app.kubernetes.io/instance=mariadb-test + tail: 50 + steps: + - name: create-mariadb + try: + - apply: + file: mariadb.yaml + + - name: wait-helmrelease-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: mariadb-test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: verify-service-and-statefulset + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: mariadb-test + spec: + (ports[0].port): 3306 + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: mariadb-test + (subsets[0].addresses[0].ip != null): true + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: mariadb-test + status: + replicas: 2 + + - name: verify-metrics + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: mariadb-test-metrics + spec: + (ports[0].port): 9104 + - assert: + timeout: 1m + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: mariadb-test-metrics + (subsets[0].addresses[0].ip != null): true + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: mariadb-test-metrics + status: + replicas: 1 diff --git a/hack/e2e-chainsaw/mariadb/mariadb.yaml b/hack/e2e-chainsaw/mariadb/mariadb.yaml new file mode 100644 index 000000000..95c46d3df --- /dev/null +++ b/hack/e2e-chainsaw/mariadb/mariadb.yaml @@ -0,0 +1,29 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: MariaDB +metadata: + name: test +spec: + external: false + size: 10Gi + replicas: 2 + storageClass: "" + users: + testuser: + maxUserConnections: 1000 + password: xai7Wepo + databases: + testdb: + roles: + admin: + - testuser + backup: + enabled: false + s3Region: us-east-1 + s3Bucket: s3.example.org/mariadb-backups + schedule: "0 2 * * *" + cleanupStrategy: "--keep-last=3 --keep-daily=3 --keep-within-weekly=1m" + s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu + s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog + resticPassword: ChaXoveekoh6eigh4siesheeda2quai0 + resources: {} + resourcesPreset: "nano" diff --git a/hack/e2e-chainsaw/mongodb/chainsaw-test.yaml b/hack/e2e-chainsaw/mongodb/chainsaw-test.yaml new file mode 100644 index 000000000..aada6d5c9 --- /dev/null +++ b/hack/e2e-chainsaw/mongodb/chainsaw-test.yaml @@ -0,0 +1,62 @@ +# Chainsaw port of hack/e2e-apps/mongodb.bats ("Create DB MongoDB"). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: mongodb +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: mongodb-test + - podLogs: + selector: app.kubernetes.io/instance=mongodb-test + tail: 50 + steps: + - name: create-mongodb + try: + - apply: + file: mongodb.yaml + + - name: wait-helmrelease-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: mongodb-test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: verify-service-and-statefulset + try: + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Service + metadata: + name: mongodb-test-rs0 + spec: + (ports[0].port): 27017 + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: mongodb-test-rs0 + (subsets[0].addresses[0].ip != null): true + - assert: + timeout: 5m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: mongodb-test-rs0 + status: + replicas: 1 diff --git a/hack/e2e-chainsaw/mongodb/mongodb.yaml b/hack/e2e-chainsaw/mongodb/mongodb.yaml new file mode 100644 index 000000000..8aabed3cd --- /dev/null +++ b/hack/e2e-chainsaw/mongodb/mongodb.yaml @@ -0,0 +1,20 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: MongoDB +metadata: + name: test +spec: + external: false + size: 10Gi + replicas: 1 + storageClass: "" + resourcesPreset: "small" + users: + testuser: + password: xai7Wepo + databases: + testdb: + roles: + admin: + - testuser + backup: + enabled: false diff --git a/hack/e2e-chainsaw/openbao/chainsaw-test.yaml b/hack/e2e-chainsaw/openbao/chainsaw-test.yaml new file mode 100644 index 000000000..be34a59e8 --- /dev/null +++ b/hack/e2e-chainsaw/openbao/chainsaw-test.yaml @@ -0,0 +1,118 @@ +# Chainsaw port of hack/e2e-apps/openbao.bats ("Create OpenBAO (standalone)"). +# +# The init/unseal sequence is inherently imperative (exec into the pod, parse +# JSON, feed the key back), so it stays a script step. Everything around it — +# HelmRelease readiness, container-started gate, StatefulSet/PVC state — is +# declarative. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: openbao +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: openbao-test + - podLogs: + selector: app.kubernetes.io/instance=openbao-test + tail: 100 + steps: + - name: create-openbao + try: + - apply: + file: openbao.yaml + + - name: wait-helmreleases-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: openbao-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: openbao-test-system + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: wait-container-started + try: + # Pod Running does not guarantee the container is ready for exec on slow CI; + # gate on containerStatuses[0].started like the BATS test did. + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Pod + metadata: + name: openbao-test-0 + (status.containerStatuses[0].started): true + + - name: init-and-unseal + try: + - script: + timeout: 3m + content: | + set -eu + ns="$NAMESPACE" + pod=openbao-test-0 + # Wait for the API to accept connections. `bao status` exit codes: + # 0 = unsealed, 2 = sealed but responsive, 1 = error/not ready. + deadline=$(( $(date +%s) + 60 )) + until rc=0; kubectl -n "$ns" exec "$pod" -- bao status >/dev/null 2>&1 || rc=$?; \ + [ "${rc:-0}" -eq 0 ] || [ "${rc:-0}" -eq 2 ]; do + [ "$(date +%s)" -ge "$deadline" ] && { echo "OpenBAO API not responsive" >&2; exit 1; } + sleep 3 + done + # Single key share for test simplicity. + init_output=$(kubectl -n "$ns" exec "$pod" -- \ + bao operator init -key-shares=1 -key-threshold=1 -format=json) + unseal_key=$(echo "$init_output" | jq -r '.unseal_keys_b64[0]') + if [ -z "$unseal_key" ] || [ "$unseal_key" = "null" ]; then + echo "Failed to extract unseal key. Init output: $init_output" >&2 + exit 1 + fi + kubectl -n "$ns" exec "$pod" -- bao operator unseal "$unseal_key" + check: + ($error == null): true + + - name: verify-statefulset-and-storage + try: + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: openbao-test + status: + readyReplicas: 1 + - assert: + timeout: 1m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data-openbao-test-0 + status: + phase: Bound + # The PVC is StatefulSet-owned (not applied by chainsaw), so delete it + # explicitly like the BATS test did. + finally: + - delete: + ref: + apiVersion: v1 + kind: PersistentVolumeClaim + name: data-openbao-test-0 diff --git a/hack/e2e-chainsaw/openbao/openbao.yaml b/hack/e2e-chainsaw/openbao/openbao.yaml new file mode 100644 index 000000000..8764c59d3 --- /dev/null +++ b/hack/e2e-chainsaw/openbao/openbao.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: OpenBAO +metadata: + name: test +spec: + replicas: 1 + size: 10Gi + storageClass: "" + resourcesPreset: "small" + resources: {} + external: false + ui: true diff --git a/hack/e2e-chainsaw/qdrant/chainsaw-test.yaml b/hack/e2e-chainsaw/qdrant/chainsaw-test.yaml new file mode 100644 index 000000000..44009b975 --- /dev/null +++ b/hack/e2e-chainsaw/qdrant/chainsaw-test.yaml @@ -0,0 +1,64 @@ +# Chainsaw port of hack/e2e-apps/qdrant.bats ("Create Qdrant"). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: qdrant +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: qdrant-test + - podLogs: + selector: app.kubernetes.io/instance=qdrant-test + tail: 50 + steps: + - name: create-qdrant + try: + - apply: + file: qdrant.yaml + + - name: wait-helmreleases-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: qdrant-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: qdrant-test-system + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: verify-statefulset-and-storage + try: + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: qdrant-test + status: + readyReplicas: 1 + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: qdrant-storage-qdrant-test-0 + status: + phase: Bound diff --git a/hack/e2e-chainsaw/qdrant/qdrant.yaml b/hack/e2e-chainsaw/qdrant/qdrant.yaml new file mode 100644 index 000000000..c91ca69d9 --- /dev/null +++ b/hack/e2e-chainsaw/qdrant/qdrant.yaml @@ -0,0 +1,11 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Qdrant +metadata: + name: test +spec: + replicas: 1 + size: 10Gi + storageClass: "" + resourcesPreset: "nano" + resources: {} + external: false diff --git a/hack/e2e-chainsaw/redis/chainsaw-test.yaml b/hack/e2e-chainsaw/redis/chainsaw-test.yaml new file mode 100644 index 000000000..d22695c30 --- /dev/null +++ b/hack/e2e-chainsaw/redis/chainsaw-test.yaml @@ -0,0 +1,64 @@ +# Chainsaw port of hack/e2e-apps/redis.bats ("Create Redis"). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: redis +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: redis-test + - podLogs: + selector: app.kubernetes.io/instance=redis-test + tail: 50 + steps: + - name: create-redis + try: + - apply: + file: redis.yaml + + - name: wait-helmrelease-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: redis-test + status: + (conditions[?type == 'Ready']): + - status: "True" + + - name: verify-storage-and-workloads + try: + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: redisfailover-persistent-data-rfr-redis-test-0 + status: + phase: Bound + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: rfs-redis-test + status: + (conditions[?type == 'Available']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: rfr-redis-test + status: + replicas: 2 diff --git a/hack/e2e-chainsaw/redis/redis.yaml b/hack/e2e-chainsaw/redis/redis.yaml new file mode 100644 index 000000000..85cf34f94 --- /dev/null +++ b/hack/e2e-chainsaw/redis/redis.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Redis +metadata: + name: test +spec: + external: false + size: 1Gi + replicas: 2 + storageClass: "" + authEnabled: true + resources: {} + resourcesPreset: "nano" diff --git a/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml new file mode 100644 index 000000000..8a9110313 --- /dev/null +++ b/hack/e2e-chainsaw/vminstance/chainsaw-test.yaml @@ -0,0 +1,137 @@ +# Chainsaw port of hack/e2e-apps/vminstance.bats. Two scenarios: a standalone +# VMDisk, and a VMInstance booting from its own VMDisk. Both download the Ubuntu +# noble cloud image, so the readiness timeouts are generous. Chainsaw deletes +# the VMDisk/VMInstance CRs it applied at cleanup. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: vmdisk +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: vm-disk-test + steps: + - name: create-vmdisk + try: + - apply: + file: vmdisk.yaml + - name: wait-vmdisk-ready + try: + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: vm-disk-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 4m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: vm-disk-test + status: + phase: Bound +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: vminstance +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: vm-instance-test + - describe: + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + name: vm-instance-test + steps: + - name: create-vmdisk + try: + - apply: + file: vmdisk-vmi.yaml + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: vm-disk-vmi-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-vmi-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 4m + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: vm-disk-vmi-test + status: + phase: Bound + - name: create-vminstance + try: + - apply: + file: vminstance.yaml + - assert: + timeout: 6m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: vm-instance-test + status: + (conditions[?type == 'Ready']): + - status: "True" + - name: wait-vm-running + try: + # Nested KubeVirt startup (virt-launcher + libvirt + cloud-init DHCP) takes + # 30-60s; allow a generous bound for the IP to appear, then VM Ready. + - assert: + timeout: 5m + resource: + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + metadata: + name: vm-instance-test + (status.interfaces[0].ipAddress != null): true + - assert: + timeout: 5m + resource: + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: vm-instance-test + status: + (conditions[?type == 'Ready']): + - status: "True" diff --git a/hack/e2e-chainsaw/vminstance/vmdisk-vmi.yaml b/hack/e2e-chainsaw/vminstance/vmdisk-vmi.yaml new file mode 100644 index 000000000..baa4814c8 --- /dev/null +++ b/hack/e2e-chainsaw/vminstance/vmdisk-vmi.yaml @@ -0,0 +1,11 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: VMDisk +metadata: + name: vmi-test +spec: + source: + http: + url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img + optical: false + storage: 5Gi + storageClass: replicated diff --git a/hack/e2e-chainsaw/vminstance/vmdisk.yaml b/hack/e2e-chainsaw/vminstance/vmdisk.yaml new file mode 100644 index 000000000..0ef5bd6d9 --- /dev/null +++ b/hack/e2e-chainsaw/vminstance/vmdisk.yaml @@ -0,0 +1,11 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: VMDisk +metadata: + name: test +spec: + source: + http: + url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img + optical: false + storage: 5Gi + storageClass: replicated diff --git a/hack/e2e-chainsaw/vminstance/vminstance.yaml b/hack/e2e-chainsaw/vminstance/vminstance.yaml new file mode 100644 index 000000000..013028744 --- /dev/null +++ b/hack/e2e-chainsaw/vminstance/vminstance.yaml @@ -0,0 +1,28 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: VMInstance +metadata: + name: test +spec: + external: false + externalMethod: PortList + externalPorts: + - 22 + running: true + instanceType: "u1.medium" + instanceProfile: ubuntu + disks: + - name: vmi-test + gpus: [] + sshKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF + test@test + cloudInit: | + #cloud-config + users: + - name: test + shell: /bin/bash + sudo: ['ALL=(ALL) NOPASSWD: ALL'] + groups: sudo + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test + cloudInitSeed: "" diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats index 092fe06d9..2671bfd63 100644 --- a/hack/remediation-guard.bats +++ b/hack/remediation-guard.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats # ----------------------------------------------------------------------------- -# Unit tests for hack/e2e-apps/remediation-guard.sh +# Unit tests for hack/e2e-chainsaw/_lib/remediation-guard.sh # # helmrelease_has_remediation_cycle takes a newline-delimited list of # HelmRelease history snapshot status values (deployed/superseded/failed/ @@ -20,7 +20,7 @@ # ----------------------------------------------------------------------------- @test "empty history returns not-detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh if helmrelease_has_remediation_cycle ""; then echo "expected not-detected for empty history" >&2 exit 1 @@ -28,7 +28,7 @@ } @test "single deployed snapshot returns not-detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh if helmrelease_has_remediation_cycle "deployed"; then echo "expected not-detected for deployed-only history" >&2 exit 1 @@ -36,7 +36,7 @@ } @test "deployed then superseded returns not-detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh statuses=$(printf 'deployed\nsuperseded\n') if helmrelease_has_remediation_cycle "${statuses}"; then echo "expected not-detected for deployed+superseded history" >&2 @@ -45,7 +45,7 @@ } @test "single failed snapshot returns detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh if ! helmrelease_has_remediation_cycle "failed"; then echo "expected detected when history contains failed snapshot" >&2 exit 1 @@ -56,7 +56,7 @@ # The exact signature of the install-remediation race: the first install # exceeded flux's wait budget, remediation uninstalled, the next retry # eventually succeeded. History still carries the uninstalled snapshot. - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh if ! helmrelease_has_remediation_cycle "uninstalled"; then echo "expected detected when history contains uninstalled snapshot" >&2 exit 1 @@ -64,7 +64,7 @@ } @test "uninstalled then deployed still returns detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh statuses=$(printf 'uninstalled\ndeployed\n') if ! helmrelease_has_remediation_cycle "${statuses}"; then echo "expected detected despite later successful deploy" >&2 @@ -73,7 +73,7 @@ } @test "deployed then failed still returns detected" { - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh statuses=$(printf 'deployed\nfailed\n') if ! helmrelease_has_remediation_cycle "${statuses}"; then echo "expected detected when any entry is failed" >&2 @@ -119,7 +119,7 @@ YAML [ -n "$statuses" ] echo "$statuses" | grep --quiet '^uninstalled$' - . hack/e2e-apps/remediation-guard.sh + . hack/e2e-chainsaw/_lib/remediation-guard.sh if ! helmrelease_has_remediation_cycle "$statuses"; then echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2 exit 1 diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index d8924ae93..d85b46d3c 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -1,32 +1,35 @@ #!/bin/sh -# Read a list of changed files (one per line) and emit space-separated app -# names whose bats files in hack/e2e-apps/ should run. +# Read a list of changed files (one per line) and emit space-separated suite +# names whose Chainsaw suites under hack/e2e-chainsaw/ should run. # # Usage: hack/select-e2e.sh [] # Defaults: sources-dir = packages/core/platform/sources # # Output: -# - empty no E2E impact (docs / dashboards / *.md only) -# - selected per the PackageSource dependency graph -# - full list any path that affects all tests, OR an unrecognised -# packages/* path, OR a per-app source whose graph has -# no *-application descendants (conservative fallback) +# - empty no E2E impact (docs / dashboards / *.md only) +# - selected per the PackageSource dependency graph +# - full list any path that affects all tests, OR an unrecognised +# packages/* path, OR a per-app source whose graph has +# no *-application descendants (conservative fallback) set -eu CHANGED="${1:?missing changed-files arg}" SOURCES_DIR="${2:-packages/core/platform/sources}" -# Anything matching this pattern triggers the full bats suite. Per-app bats -# (hack/e2e-apps/.bats) are matched BEFORE this so editing one bats -# file doesn't escalate to the full suite. -full_suite_pattern='^(packages/library/|packages/core/|api/|cmd/|internal/|hack/[^/]+\.sh$|hack/[^/]+\.bats$|hack/e2e-apps/[^/]+\.sh$|Makefile$|\.github/workflows/pull-requests\.yaml$)' +# Anything matching this pattern triggers the full Chainsaw suite. Per-suite +# edits (hack/e2e-chainsaw//...) are matched BEFORE this so editing one +# suite doesn't escalate to the full suite; the shared _lib helpers and the +# .chainsaw.yaml config affect every suite and DO escalate (handled inline). +full_suite_pattern='^(packages/library/|packages/core/|api/|cmd/|internal/|hack/[^/]+\.sh$|hack/[^/]+\.bats$|Makefile$|\.github/workflows/pull-requests\.yaml$)' -# All known per-app bats files -all_apps=$(ls hack/e2e-apps/*.bats 2>/dev/null | xargs -n1 basename | sed 's/\.bats$//') +# All known Chainsaw suites: every dir under hack/e2e-chainsaw/ holding a +# chainsaw-test.yaml (this excludes _lib/ and the top-level config files). +all_apps=$(find hack/e2e-chainsaw -mindepth 2 -maxdepth 2 -name chainsaw-test.yaml 2>/dev/null \ + | sed -e 's,^hack/e2e-chainsaw/,,' -e 's,/chainsaw-test\.yaml$,,' | sort) -# PackageSource name -> bats name(s). Most *-application sources map by -# stripping the suffix; explicit overrides for the few that don't. -app_to_bats() { +# PackageSource name -> Chainsaw suite name(s). Most *-application sources map +# by stripping the suffix; explicit overrides for the few that don't. +src_to_suites() { case "$1" in postgres-application) echo postgres ;; vm-instance-application) echo vminstance ;; @@ -63,13 +66,19 @@ while IFS= read -r file; do continue fi - # 2. Per-app bats first — editing one bats file selects only that app. - if echo "$file" | grep -qE '^hack/e2e-apps/[^/]+\.bats$'; then - app=$(basename "$file" .bats) - selected_apps="$selected_apps $app" - trigger_any=1 - continue - fi + # 2. Chainsaw edits. A per-suite file selects only that suite; the shared + # _lib helpers and the .chainsaw.yaml config affect every suite, so they + # escalate to the full run. + case "$file" in + hack/e2e-chainsaw/_lib/*|hack/e2e-chainsaw/.chainsaw.yaml) + trigger_full=1 + continue ;; + hack/e2e-chainsaw/*/*) + app=$(echo "$file" | sed -nE 's,^hack/e2e-chainsaw/([^/]+)/.*,\1,p') + selected_apps="$selected_apps $app" + trigger_any=1 + continue ;; + esac # 3. Full-suite trigger if echo "$file" | grep -qE "$full_suite_pattern"; then @@ -116,20 +125,20 @@ while :; do all_sources="$all_sources $new" done -# Filter to *-application sources, then map to bats names. +# Filter to *-application sources, then map to Chainsaw suite names. final="" for s in $all_sources; do app=${s#cozystack.} case "$app" in - *-application) final="$final $(app_to_bats "$app")" ;; + *-application) final="$final $(src_to_suites "$app")" ;; external-dns) final="$final external-dns" ;; esac done -# Add directly-selected apps from per-app bats edits. +# Add directly-selected suites from per-suite edits. final="$final $selected_apps" -# Deduplicate; intersect with available bats files. +# Deduplicate; intersect with available Chainsaw suites. final_apps=$(echo "$final" | tr ' ' '\n' | sort -u | grep -v '^$' | while read -r app; do if echo "$all_apps" | grep -Fxq "$app"; then echo "$app" diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index 192566823..c0c54fc52 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -11,7 +11,7 @@ # Run with: hack/cozytest.sh hack/select-e2e_test.bats # ----------------------------------------------------------------------------- -@test "single app diff selects only that bats" { +@test "single app diff selects only that suite" { tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT cp -r packages/core/platform/sources "$tmp/sources" @@ -20,10 +20,10 @@ [ "$output" = "postgres" ] } -@test "operator diff selects all dependent app bats" { +@test "operator diff selects all dependent app suites" { # postgres-operator is depended on by postgres-application, harbor-application # (Harbor uses postgres as its backing DB), and monitoring-application (Grafana - # DB). monitoring isn't in hack/e2e-apps/ so it's filtered out by the selector. + # DB). monitoring has no chainsaw suite so it's filtered out by the selector. tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT cp -r packages/core/platform/sources "$tmp/sources" @@ -43,7 +43,7 @@ cp -r packages/core/platform/sources "$tmp/sources" echo "packages/system/cilium/values.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") - # Full suite means more than 5 bats files + # Full suite means more than 5 suites [ "$(echo "$output" | wc -w)" -gt 5 ] } @@ -65,7 +65,7 @@ [ -z "$output" ] } -@test "kubernetes-application maps to two bats files" { +@test "kubernetes-application maps to two suites" { tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT cp -r packages/core/platform/sources "$tmp/sources" @@ -88,7 +88,16 @@ tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT cp -r packages/core/platform/sources "$tmp/sources" - echo "hack/e2e-apps/run-kubernetes.sh" > "$tmp/diff" + echo "hack/e2e-chainsaw/_lib/run-kubernetes.sh" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ "$(echo "$output" | wc -w)" -gt 5 ] +} + +@test "chainsaw config change triggers full suite" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + cp -r packages/core/platform/sources "$tmp/sources" + echo "hack/e2e-chainsaw/.chainsaw.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") [ "$(echo "$output" | wc -w)" -gt 5 ] } @@ -102,11 +111,11 @@ [ "$(echo "$output" | wc -w)" -gt 5 ] } -@test "per-app bats edit selects only that app, never escalates" { +@test "per-suite edit selects only that suite, never escalates" { tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT cp -r packages/core/platform/sources "$tmp/sources" - echo "hack/e2e-apps/redis.bats" > "$tmp/diff" + echo "hack/e2e-chainsaw/redis/chainsaw-test.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") [ "$output" = "redis" ] } diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile index d458ed763..58dbfa7a1 100644 --- a/packages/core/testing/Makefile +++ b/packages/core/testing/Makefile @@ -24,7 +24,11 @@ image-e2e-sandbox: yq -i '.e2e.image = strenv(IMAGE)' values.yaml rm -f images/e2e-sandbox.json -test: test-openapi test-apps ## Run the end-to-end tests in existing sandbox +# Named `e2e` (not `test`) so hack/helm-unit-tests.sh — which runs the `test` +# target of every package whose `make -n test` resolves — does not pick up this +# sandbox-driven e2e flow and try to run it in the unit-test phase (no sandbox +# exists then). The root `make test` target invokes this. +e2e: test-openapi test-chainsaw ## Run the end-to-end tests in existing sandbox copy-nocloud-image: docker cp ../../../_out/assets/nocloud-amd64.raw.xz "${SANDBOX_NAME}":/workspace/_out/assets/nocloud-amd64.raw.xz @@ -38,8 +42,14 @@ install-cozystack: test-openapi: docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-test-openapi.bats' -test-apps-%: - docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-apps/$*.bats' +# Run the Chainsaw E2E app suite. Chainsaw loads .chainsaw.yaml from the cwd, +# discovers every chainsaw-test.yaml under hack/e2e-chainsaw/, polls each +# assertion to its own timeout, and writes a JUnit chainsaw-report.xml the CI +# job uploads. CHAINSAW_SUITES restricts the run to the named suite dirs +# (space-separated, e.g. "postgres bucket"); empty runs the whole suite. The +# CI Test-Impact-Analysis step (hack/select-e2e.sh) sets it from the PR diff. +test-chainsaw: + docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace/hack/e2e-chainsaw && chainsaw test ${CHAINSAW_SUITES}' collect-report: ## Collect the test report from the sandbox. docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozyreport.sh cozyreport' diff --git a/packages/core/testing/images/e2e-sandbox/Dockerfile b/packages/core/testing/images/e2e-sandbox/Dockerfile index ca7db8e15..4c1fc68b1 100644 --- a/packages/core/testing/images/e2e-sandbox/Dockerfile +++ b/packages/core/testing/images/e2e-sandbox/Dockerfile @@ -5,6 +5,7 @@ ARG TALOSCTL_VERSION=1.13.5 ARG HELM_VERSION=3.21.2 ARG COZYHR_VERSION=1.6.1 ARG CRUST_GATHER_VERSION=v0.15.2 +ARG CHAINSAW_VERSION=0.2.15 ARG TARGETOS ARG TARGETARCH @@ -26,6 +27,9 @@ RUN curl -fsSL "https://dl.min.io/client/mc/release/${TARGETOS}-${TARGETARCH}/mc # cozytest.sh (host + each nested tenant cluster) and `crust-gather serve`-able # from the cozyreport artifact. Installs a static musl binary as crust-gather. RUN curl -sSfL "https://github.com/crust-gather/crust-gather/raw/main/install.sh" | sh -s -- --version "${CRUST_GATHER_VERSION}" --bin-dir /usr/local/bin --yes +RUN curl -sSL "https://github.com/kyverno/chainsaw/releases/download/v${CHAINSAW_VERSION}/chainsaw_${TARGETOS}_${TARGETARCH}.tar.gz" \ + | tar -xz -C /usr/local/bin chainsaw \ + && chmod +x /usr/local/bin/chainsaw RUN <<'EOF' cat <<'EOT' >> /etc/bash.bashrc . /etc/bash_completion