mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 14:02:56 +09:00
Myasnikov Daniil
2149e59a94
Mirror the diagnostics group already on `Run E2E tests` to the `Install Cozystack into sandbox` step. Without this, install-step failures (e.g. an HR not reaching Ready inside the bats `kubectl wait` window) surface only as "error: timed out waiting for the condition on helmreleases/<name>" with no cluster snapshot — re-running is the only way to triage, which is the exact pattern the parent commit removed. On failure, dump three groups: - `kubectl get hr -A -o wide` (last 100 lines): which HRs exist and their Ready/Status columns. - `kubectl describe hr` for each not-Ready HR (tail -50): the Conditions block surfaces helm install/upgrade errors, missing CRDs, and chart-render failures — the actual cause. - `kubectl get events -A --sort-by=.lastTimestamp` (last 50): cluster events that often pinpoint the failing pod/PVC/job. Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
326 lines
11 KiB
YAML
326 lines
11 KiB
YAML
name: Pull Request
|
||
|
||
env:
|
||
# TODO: unhardcode this
|
||
REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
|
||
on:
|
||
pull_request:
|
||
types: [opened, synchronize, reopened]
|
||
|
||
# Cancel in‑flight runs for the same PR when a new push arrives.
|
||
concurrency:
|
||
group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
detect-changes:
|
||
name: Detect changes
|
||
runs-on: ubuntu-latest
|
||
outputs:
|
||
code: ${{ steps.filter.outputs.code }}
|
||
steps:
|
||
- uses: dorny/paths-filter@v3
|
||
id: filter
|
||
with:
|
||
filters: |
|
||
code:
|
||
- '!docs/**'
|
||
|
||
build:
|
||
name: Build
|
||
runs-on: [self-hosted]
|
||
timeout-minutes: 30
|
||
permissions:
|
||
contents: read
|
||
packages: write
|
||
|
||
needs: ["detect-changes"]
|
||
# Never run when the PR carries the "release" label or only docs changed.
|
||
if: |
|
||
needs.detect-changes.outputs.code == 'true'
|
||
&& !contains(github.event.pull_request.labels.*.name, 'release')
|
||
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
|
||
- name: Run unit tests
|
||
run: make unit-tests
|
||
|
||
- name: Set up Docker config
|
||
run: |
|
||
if [ -d ~/.docker ]; then
|
||
cp -r ~/.docker "${{ runner.temp }}/.docker"
|
||
fi
|
||
|
||
- name: Login to GitHub Container Registry
|
||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||
uses: docker/login-action@v3
|
||
with:
|
||
username: ${{ secrets.OCIR_USER}}
|
||
password: ${{ secrets.OCIR_TOKEN }}
|
||
registry: iad.ocir.io
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
|
||
- name: Build
|
||
run: make build
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
|
||
- name: Build Talos image
|
||
run: make -C packages/core/talos talos-nocloud
|
||
|
||
- name: Save git diff as patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
run: git diff HEAD > _out/assets/pr.patch
|
||
|
||
- name: Upload git diff patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: pr-patch
|
||
path: _out/assets/pr.patch
|
||
|
||
- name: Upload Talos image
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: talos-image
|
||
path: _out/assets/nocloud-amd64.raw.xz
|
||
|
||
resolve_assets:
|
||
name: "Resolve assets"
|
||
runs-on: ubuntu-latest
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
outputs:
|
||
disk_id: ${{ steps.fetch_assets.outputs.disk_id }}
|
||
|
||
steps:
|
||
- name: Generate GitHub App token
|
||
id: app-token
|
||
uses: actions/create-github-app-token@v1
|
||
with:
|
||
app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
|
||
private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
|
||
owner: cozystack
|
||
|
||
- name: Checkout code
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
uses: actions/checkout@v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
|
||
- name: Extract tag from PR branch (release PR)
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
id: get_tag
|
||
uses: actions/github-script@v7
|
||
with:
|
||
script: |
|
||
const branch = context.payload.pull_request.head.ref;
|
||
const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
|
||
if (!m) {
|
||
core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
|
||
return;
|
||
}
|
||
core.setOutput('tag', `v${m[1]}`);
|
||
|
||
- name: Find draft release & asset IDs (release PR)
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
id: fetch_assets
|
||
uses: actions/github-script@v7
|
||
with:
|
||
github-token: ${{ steps.app-token.outputs.token }}
|
||
script: |
|
||
const tag = '${{ steps.get_tag.outputs.tag }}';
|
||
const releases = await github.rest.repos.listReleases({
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
per_page: 100
|
||
});
|
||
const draft = releases.data.find(r => r.tag_name === tag && r.draft);
|
||
if (!draft) {
|
||
core.setFailed(`Draft release '${tag}' not found`);
|
||
return;
|
||
}
|
||
const find = (n) => draft.assets.find(a => a.name === n)?.id;
|
||
const diskId = find('nocloud-amd64.raw.xz');
|
||
if (!diskId) {
|
||
core.setFailed('Required assets missing in draft release');
|
||
return;
|
||
}
|
||
core.setOutput('disk_id', diskId);
|
||
|
||
|
||
e2e:
|
||
name: "E2E Tests"
|
||
runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }}
|
||
#runs-on: [oracle-vm-32cpu-128gb-x86-64]
|
||
timeout-minutes: 180
|
||
permissions:
|
||
contents: read
|
||
packages: read
|
||
needs: ["build", "resolve_assets"]
|
||
if: ${{ always() && (needs.build.result == 'success' || needs.resolve_assets.result == 'success') }}
|
||
|
||
steps:
|
||
- name: Generate GitHub App token
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
id: app-token
|
||
uses: actions/create-github-app-token@v1
|
||
with:
|
||
app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
|
||
private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
|
||
owner: cozystack
|
||
|
||
# ▸ Checkout and prepare the codebase
|
||
- name: Checkout code
|
||
uses: actions/checkout@v4
|
||
|
||
# ▸ Regular PR path – download artefacts produced by the *build* job
|
||
- name: "Download Talos image (regular PR)"
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
uses: actions/download-artifact@v4
|
||
with:
|
||
name: talos-image
|
||
path: _out/assets
|
||
|
||
- name: Download PR patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
uses: actions/download-artifact@v4
|
||
with:
|
||
name: pr-patch
|
||
path: _out/assets
|
||
|
||
- name: Apply patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
run: |
|
||
git apply _out/assets/pr.patch
|
||
|
||
# ▸ Release PR path – fetch artefacts from the corresponding draft release
|
||
- name: Download assets from draft release (release PR)
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
run: |
|
||
mkdir -p _out/assets
|
||
curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \
|
||
-o _out/assets/nocloud-amd64.raw.xz \
|
||
"https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}"
|
||
env:
|
||
APP_TOKEN: ${{ steps.app-token.outputs.token }}
|
||
|
||
- name: Set sandbox ID
|
||
run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV
|
||
|
||
# ▸ Prepare environment
|
||
- name: Prepare workspace
|
||
run: |
|
||
rm -rf /tmp/$SANDBOX_NAME
|
||
cp -r ${{ github.workspace }} /tmp/$SANDBOX_NAME
|
||
|
||
- name: Prepare environment
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
attempt=0
|
||
until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
|
||
attempt=$((attempt + 1))
|
||
if [ $attempt -ge 3 ]; then
|
||
echo "Attempt $attempt failed, exiting..."
|
||
exit 1
|
||
fi
|
||
echo "Attempt $attempt failed, retrying..."
|
||
done
|
||
echo "Prepare environment completed after $((attempt + 1)) attempts"
|
||
|
||
# ▸ Install Cozystack
|
||
- name: Install Cozystack into sandbox
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
|
||
echo "❌ Install failed (no retry — see diagnostics below)"
|
||
echo "::group::Diagnostics: HelmRelease status"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
|
||
echo "::endgroup::"
|
||
echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
|
||
| while read -r ns name; do
|
||
echo "--- $ns/$name ---"
|
||
docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
|
||
done
|
||
echo "::endgroup::"
|
||
echo "::group::Diagnostics: recent events"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
|
||
echo "::endgroup::"
|
||
exit 1
|
||
fi
|
||
|
||
- name: Run OpenAPI tests
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi
|
||
|
||
# ▸ Run E2E tests
|
||
- name: Run E2E tests
|
||
id: e2e_tests
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
failed_tests=""
|
||
for app in $(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1); do
|
||
echo "::group::Testing $app"
|
||
if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
|
||
echo "::endgroup::"
|
||
echo "✅ Test $app completed successfully"
|
||
else
|
||
echo "::endgroup::"
|
||
echo "❌ Test $app failed (no retry — see diagnostics below)"
|
||
failed_tests="$failed_tests $app"
|
||
echo "::group::Diagnostics for $app"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
|
||
echo "::endgroup::"
|
||
fi
|
||
done
|
||
if [ -n "$failed_tests" ]; then
|
||
echo "❌ Failed tests:$failed_tests"
|
||
exit 1
|
||
fi
|
||
echo "✅ All E2E tests passed"
|
||
|
||
# ▸ Collect debug information (always runs)
|
||
- name: Collect report
|
||
if: always()
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true
|
||
|
||
- name: Upload cozyreport.tgz
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: cozyreport
|
||
path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz
|
||
|
||
- name: Collect images list
|
||
if: always()
|
||
run: |
|
||
cd /tmp/$SANDBOX_NAME
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true
|
||
|
||
- name: Upload image list
|
||
if: always()
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: image-list
|
||
path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt
|
||
|
||
# ▸ Tear down environment (always runs)
|
||
- name: Tear down sandbox
|
||
if: always()
|
||
run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true
|
||
|
||
- name: Remove workspace
|
||
if: always()
|
||
run: rm -rf /tmp/$SANDBOX_NAME
|