NAMESPACE=cozy-e2e-tests NAME := sandbox CLEAN := 1 TESTING_APPS := $(shell find ../../apps -maxdepth 1 -mindepth 1 -type d | awk -F/ '{print $$NF}') SANDBOX_NAME := cozy-e2e-sandbox-$(shell echo "$$(hostname):$$(pwd)" | sha256sum | cut -c -6) COZY_E2E_NODE_CPUS ?= 8 COZY_E2E_NODE_MEMORY_MIB ?= 24576 ROOT_DIR = $(dir $(abspath $(firstword $(MAKEFILE_LIST))/../../..)) # One container lane per host. Unlike SANDBOX_NAME above, these names are not # per-checkout, and hack/e2e-compose.yaml's `container_name: srv1..3` and the # `data-srvN` zpools are global besides -- so a second checkout on the same host # adopts this one's nodes and pools, and its delete-cluster-container destroys # them. Not reachable in CI, where the lane already fills the largest runner # shape (3 x 8 vCPU / 24 GiB against 32 / 128), and left as a known limit rather # than fixed because isolating it properly means renaming the pools, which the # LINSTOR storage-pool registration reads. COMPOSE_PROJECT ?= cozy-e2e COMPOSE_FILE ?= $(ROOT_DIR)hack/e2e-compose.yaml ZPOOL_BACKING_DIR ?= /var/lib/cozy-e2e-zpools # The e2e sandbox image is pinned to amd64 as a whole: its VM lane boots the # amd64 Talos disk under qemu-system-x86_64 with KVM # (hack/e2e-prepare-cluster.bats), which an arm64 sandbox cannot run. # `override` beats any PLATFORM given on the command line, an explicit # linux/arm64 as much as one passed to a whole `make build`. It must precede # the include: common-envs.mk expands BUILDX_ARGS immediately. override PLATFORM := linux/amd64 include ../../../hack/common-envs.mk help: ## Show this help. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) image: image-e2e-sandbox image-e2e-sandbox: docker buildx build -f images/e2e-sandbox/Dockerfile images/e2e-sandbox \ $(call image-tags,e2e-sandbox,v$(COZYSTACK_VERSION)) \ $(call cache-args,e2e-sandbox) \ --metadata-file images/e2e-sandbox.json \ $(BUILDX_ARGS) IMAGE="$(REGISTRY)/e2e-sandbox:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/e2e-sandbox.json -o json -r)" \ yq -i '.e2e.image = strenv(IMAGE)' values.yaml rm -f images/e2e-sandbox.json # Named `e2e` (not `test`) so hack/helm-unit-tests.sh — which runs the `test` # target of every package whose `make -n test` resolves — does not pick up this # sandbox-driven e2e flow and try to run it in the unit-test phase (no sandbox # exists then). The root `make test` target invokes this. e2e: test-openapi test-chainsaw ## Run the end-to-end tests in existing sandbox copy-nocloud-image: docker cp ../../../_out/assets/nocloud-amd64.raw.xz "${SANDBOX_NAME}":/workspace/_out/assets/nocloud-amd64.raw.xz # Live BATS suites need a literal invocation here: the root unit glob excludes # hack/e2e-* and subdirectories. hack/bats-runner-coverage.bats checks for files # with no runner and for runner paths whose files no longer exist. prepare-cluster: copy-nocloud-image docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-prepare-cluster.bats' # Container lane: Talos nodes as containers instead of QEMU guests, so tenant # workloads sit one nesting level shallower. Sits BESIDE prepare-cluster rather # than replacing it; nothing downstream changes, because the addressing, CPU and # memory in hack/e2e-compose.yaml mirror the QEMU lane exactly. # # Two steps, not one, because the halves must run in different places. The nodes # have to be siblings on the runner's docker daemon for the /dev and /run/udev # binds in hack/e2e-compose.yaml to resolve against the real host devtmpfs -- # that bind is what lets this lane carry LINSTOR at all -- so the host-side half # needs a docker socket the sandbox does not have. Everything Talos-side then # runs inside the sandbox exactly as the QEMU lane does. # # No copy-nocloud-image dependency: containers boot from the Talos image # directly and need no nocloud asset, which also means this lane does not wait # on `make -C packages/core/talos assets`. prepare-cluster-container: delete-cluster-container SANDBOX_NAME="${SANDBOX_NAME}" COMPOSE_PROJECT="${COMPOSE_PROJECT}" COMPOSE_FILE="${COMPOSE_FILE}" ZPOOL_BACKING_DIR="${ZPOOL_BACKING_DIR}" COZY_E2E_NODE_CPUS="${COZY_E2E_NODE_CPUS}" COZY_E2E_NODE_MEMORY_MIB="${COZY_E2E_NODE_MEMORY_MIB}" $(ROOT_DIR)hack/e2e-container-up.sh docker exec -e COZY_E2E_NODE_CPUS="${COZY_E2E_NODE_CPUS}" -e COZY_E2E_NODE_MEMORY_MIB="${COZY_E2E_NODE_MEMORY_MIB}" "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-prepare-cluster-container.bats' # Tear the container lane down. Docker volumes hold etcd and kubelet state, and # the host zpools hold LINSTOR data, so both halves must disappear before a # retry or the next run can adopt a half-dead cluster. # USERDATA is set to a placeholder because compose interpolates the whole file # before it decides what the subcommand needs, so the `${USERDATA:?}` guard that # makes a missing machine config fail loudly on `up` also fails `down`, where no # machine config is required. The value is never read: `down` matches containers # by project and service name. Without it teardown always errored, and ignoring # that error left the nodes and their /var volumes alive for the next run -- the # exact half-dead cluster this target exists to prevent. delete-cluster-container: @set -u; \ cleanup_rc=0; \ if USERDATA=teardown docker compose -p "${COMPOSE_PROJECT}" -f "${COMPOSE_FILE}" down -v; then \ :; \ else \ step_rc=$$?; \ echo "container compose teardown failed with exit $$step_rc; continuing host-storage cleanup" >&2; \ cleanup_rc=$$step_rc; \ fi; \ if [ -z "${ZPOOL_BACKING_DIR}" ] || [ "${ZPOOL_BACKING_DIR}" = / ]; then \ echo "refusing unsafe ZPOOL_BACKING_DIR '${ZPOOL_BACKING_DIR}'" >&2; \ exit 1; \ fi; \ if [ "$$(id -u)" -eq 0 ]; then sudo_cmd=; else sudo_cmd=sudo; fi; \ for n in 1 2 3; do \ pool_detached=1; \ if $$sudo_cmd zpool list "data-srv$${n}" >/dev/null 2>&1; then \ if $$sudo_cmd zpool destroy "data-srv$${n}"; then \ :; \ else \ step_rc=$$?; \ echo "failed to destroy zpool data-srv$${n} (exit $$step_rc)" >&2; \ if [ "$$cleanup_rc" -eq 0 ]; then cleanup_rc=$$step_rc; fi; \ pool_detached=0; \ fi; \ else \ direct_list_rc=$$?; \ inventory_rc=0; \ pool_names=$$($$sudo_cmd zpool list -H -o name 2>/dev/null) || inventory_rc=$$?; \ if [ "$$inventory_rc" -ne 0 ]; then \ echo "failed to inventory zpools after data-srv$${n} lookup returned exit $$direct_list_rc (inventory exit $$inventory_rc); keeping its backing file" >&2; \ if [ "$$cleanup_rc" -eq 0 ]; then cleanup_rc=$$inventory_rc; fi; \ pool_detached=0; \ elif printf '%s\n' "$$pool_names" | grep -Fxq "data-srv$${n}"; then \ echo "zpool inventory still lists data-srv$${n} after its direct lookup returned exit $$direct_list_rc; keeping its backing file" >&2; \ if [ "$$cleanup_rc" -eq 0 ]; then cleanup_rc=$$direct_list_rc; fi; \ pool_detached=0; \ fi; \ fi; \ if [ "$$pool_detached" -eq 0 ]; then \ echo "keeping ${ZPOOL_BACKING_DIR}/srv$${n}.img because data-srv$${n} is still imported" >&2; \ else \ if $$sudo_cmd rm -f "${ZPOOL_BACKING_DIR}/srv$${n}.img"; then \ :; \ else \ step_rc=$$?; \ echo "failed to remove ${ZPOOL_BACKING_DIR}/srv$${n}.img (exit $$step_rc)" >&2; \ if [ "$$cleanup_rc" -eq 0 ]; then cleanup_rc=$$step_rc; fi; \ fi; \ fi; \ done; \ if [ -d "${ZPOOL_BACKING_DIR}" ]; then \ if $$sudo_cmd rmdir "${ZPOOL_BACKING_DIR}"; then \ :; \ else \ step_rc=$$?; \ echo "failed to remove backing directory ${ZPOOL_BACKING_DIR} (exit $$step_rc)" >&2; \ if [ "$$cleanup_rc" -eq 0 ]; then cleanup_rc=$$step_rc; fi; \ fi; \ fi; \ exit "$$cleanup_rc" # The API endpoint and E2E-only storage mode cross into the sandbox explicitly: # docker exec starts a fresh environment, so values exported by the caller do # not reach the manifest generator or root-tenant setup. Empty values retain the # QEMU defaults; the container lane supplies its node endpoint, disables DRBD, # and selects local storage because its Talos nodes share one kernel. # COZY_DISABLED_PACKAGES is the space-separated output of # hack/select-install.sh --disabled; empty installs the whole platform. install-cozystack: docker exec -e COZY_APISERVER_ENDPOINT="${COZY_APISERVER_ENDPOINT}" -e COZY_LINSTOR_DRBD_ENABLED="${COZY_LINSTOR_DRBD_ENABLED}" -e COZY_E2E_STORAGE_CLASS="${COZY_E2E_STORAGE_CLASS}" -e COZY_DISABLED_PACKAGES="${COZY_DISABLED_PACKAGES}" "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-install-cozystack.bats' test-openapi: docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-test-openapi.bats' # Run the Chainsaw E2E app suite. Chainsaw loads .chainsaw.yaml from the cwd, # discovers each Test in or below a suite dir of hack/e2e-chainsaw/, polls each # assertion to its own timeout, and writes a JUnit chainsaw-report.xml the CI # job uploads. CHAINSAW_SUITES restricts the run to the named suite dirs # (space-separated, e.g. "postgres bucket"); empty runs the whole suite. The # CI Test-Impact-Analysis step (hack/select-e2e.sh) sets it from the PR diff. # hack/e2e-chainsaw-run.sh runs one `chainsaw test` per suite directory, so it # can fail the suite whose pods outlive its cleanup, and merges the per-suite # reports into that same chainsaw-report.xml. # COZY_E2E_STORAGE_CLASS is passed both to script operations and as a Chainsaw # value for templated resource files; empty keeps the QEMU default, replicated. test-chainsaw: docker exec -e COZY_E2E_STORAGE_CLASS="${COZY_E2E_STORAGE_CLASS}" "${SANDBOX_NAME}" sh -c 'cd /workspace/hack/e2e-chainsaw && ../e2e-chainsaw-run.sh --set-string storageClass="$${COZY_E2E_STORAGE_CLASS:-replicated}" ${CHAINSAW_SUITES}' collect-report: ## Collect the test report from the sandbox. docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozyreport.sh cozyreport' mkdir -p ../../../_out docker cp "${SANDBOX_NAME}:/workspace/cozyreport.tgz" ../../../_out/cozyreport.tgz collect-images: ## Collect the list of images used in the sandbox. docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/collect-images.sh' mkdir -p ../../../_out docker cp "${SANDBOX_NAME}":/workspace/images.txt ../../../_out/images.txt docker cp "${SANDBOX_NAME}":/workspace/image-refs.txt ../../../_out/image-refs.txt # Install for the isp-hosted lane (.github/workflows/e2e-isp-hosted.yaml): the # hosted installer variant and the isp-hosted platform variant, on a kind # cluster the workflow attaches the sandbox to. Nothing to pass in, unlike # install-cozystack: the endpoint and storage belong to the cluster. install-cozystack-hosted: docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-install-cozystack-hosted.bats' delete: ## Remove sandbox from existing Kubernetes cluster. docker rm -f "${SANDBOX_NAME}" || true exec: ## Opens an interactive shell in the sandbox container. docker exec -ti "${SANDBOX_NAME}" bash # Which machine this run landed on, recorded before the container exists. # # Here rather than in a workflow step for two reasons. The metadata service # answers on a link-local address the container is not guaranteed to reach, so # the read has to happen on this side of `docker run`; and the record has to be # inside the tree the `docker cp` below carries in, or hack/cozyreport.sh cannot # fold it into the artifact from inside. Every workflow that runs the suite reaches # the sandbox through `make prepare-env`, so one call here covers each of them on # both the green and the red path, with no per-workflow step to keep in step. No # count of those workflows here on purpose: the count is what goes stale, and the # funnel is the property worth stating. # # Ahead of `docker run`, so a runner whose sandbox fails to start is still # identified. Never fatal: the script says why it could not read something and # exits 0. On a machine with no metadata service -- a laptop running this target # directly -- it costs the read's own ceiling once, against a target that then # boots three QEMU guests. apply: delete COZY_RUNNER_IDENTITY_FILE="$(ROOT_DIR)_out/runner-identity.txt" $(ROOT_DIR)hack/e2e-runner-identity.sh docker run \ -d --rm --name "${SANDBOX_NAME}" --privileged \ -e TALOSCONFIG=/workspace/talosconfig \ -e KUBECONFIG=/workspace/kubeconfig \ "$$(yq .e2e.image values.yaml)" \ --timeout 30m docker cp "${ROOT_DIR}/." "${SANDBOX_NAME}":/workspace