diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md index 4b920837e..ef8b926e5 100644 --- a/.gemini/styleguide.md +++ b/.gemini/styleguide.md @@ -35,7 +35,7 @@ Similarly, never propose edits to: Each commit must follow [Conventional Commits](https://www.conventionalcommits.org/) format: `type(scope): brief description`. -Valid types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`. +Valid types: `feat`, `fix`, `security` (a fix that comes through a security advisory), `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`. Valid scopes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1e368718e..068b125b4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ ## What this PR does diff --git a/.github/labels.yml b/.github/labels.yml index c71518c43..d00744897 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -22,6 +22,7 @@ # do-not-merge/ PR merge blockers # security/ security-finding severity and status (Cozystack-specific) # size/ PR size (auto-applied) +# backport/ maintainer overrides of the backport batch rules (Cozystack-specific) # # `aliases:` lets EndBug/label-sync rename existing labels without losing # references on already-tagged issues and PRs. @@ -78,18 +79,6 @@ color: 'e11d21' description: Indicates the change introduces a breaking API or behaviour change -# kind/backport-previous has no upstream Kubernetes equivalent (k8s has no -# two-release-line model); it stays under kind/ per the file convention. -- name: kind/backport - color: 'fbca04' - description: Categorizes issue or PR as requiring a backport to the current release line - aliases: ['backport'] - -- name: kind/backport-previous - color: 'fbd876' - description: Categorizes issue or PR as requiring a backport to the previous release line - aliases: ['backport-previous'] - # ────────────────────────────────────────────── # priority/ — urgency # ────────────────────────────────────────────── @@ -411,3 +400,17 @@ - name: security/fixed color: '0e8a16' description: Fix released + +# ────────────────────────────────────────────── +# backport/ — maintainer overrides of the backport batch rules +# ────────────────────────────────────────────── + +# Read by cmd/backport-batch from the merged pull request on main; +# docs/release/backports.md describes both. +- name: backport/include + color: 'c2e0c6' + description: The backport batch takes this PR though rules 2-4 keep it out; it must still pick and pass CI + +- name: backport/skip + color: 'f9d0c4' + description: The backport batch keeps this PR out although the rules take it diff --git a/.github/workflows/backport-batch.yaml b/.github/workflows/backport-batch.yaml new file mode 100644 index 000000000..8196c7468 --- /dev/null +++ b/.github/workflows/backport-batch.yaml @@ -0,0 +1,107 @@ +name: Backport batch + +# Builds the backport batch for each line `backport-batch lines` names, the +# newest released lines and any line in its rc window, newest line first, as +# docs/release/backports.md describes. The steps live in +# hack/backport-batch-scheduled.sh, which a maintainer can also run by hand. +# +# The batch runs as the cozystack-ci App, with a token minted for this +# repository alone with Contents and Pull requests write. It has no +# workflows permission, so an original that changes .github/workflows/ +# waits on the batch for a maintainer (--no-workflow-files). Its pushes, +# unlike GITHUB_TOKEN's, start CI on the batch pull request. +# +# Repository settings it reads; none is created here: +# secret COZYSTACK_CI_APP_ID, COZYSTACK_CI_PRIVATE_KEY +# the cozystack-ci App, shared with the +# release workflows. Required. +# variable BACKPORT_ALERT_TEAM the team or person the batch's comment +# mentions, org/team or a login. +# +# A batch open since the previous run, deferred conflicts and originals that +# change workflows wait for a maintainer: the batch's comment lists them. A +# failed batch CI waits too, and shows on the batch pull request's checks. + +on: + schedule: + # Monday and Thursday: each run leaves working days to cut an rc, run its + # e2e and promote it before the weekend. 06:23 UTC is after the nightly + # and the daily sweeps. + - cron: '23 6 * * 1,4' + workflow_dispatch: {} + +permissions: {} + +concurrency: + group: backport-batch + cancel-in-progress: false + +jobs: + batch: + name: Backport batch + if: github.repository == 'cozystack/cozystack' + runs-on: ubuntu-latest + # The App token lives an hour from its mint, a few minutes in; a run takes + # minutes, so the job ends well before the token does. + timeout-minutes: 30 + permissions: + contents: read + env: + # The pipe falls back to direct on any proxy error (hack/common-envs.mk). + GOPROXY: "https://proxy.golang.org|direct" + steps: + - name: Require the cozystack-ci App + env: + APP_ID: ${{ secrets.COZYSTACK_CI_APP_ID }} + APP_KEY: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} + run: | + if [ -z "$APP_ID" ] || [ -z "$APP_KEY" ]; then + echo "::error::COZYSTACK_CI_APP_ID or COZYSTACK_CI_PRIVATE_KEY is not set. The batch pushes and opens pull requests as the cozystack-ci App only; see docs/release/backports.md#the-scheduled-run." + exit 1 + fi + + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: false + + - name: Build backport-batch + run: go build -o "$RUNNER_TEMP/backport-batch" ./cmd/backport-batch + + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 + with: + app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} + private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} + owner: cozystack + repositories: cozystack + permission-contents: write + permission-pull-requests: write + + # git pushes through gh, which reads the token from the environment of + # each call, so the token is never written into the checkout. + - name: Use the App's bot as git identity and credential + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + run: | + gh auth setup-git + id=$(gh api "/users/${APP_SLUG}[bot]" --jq .id) + git config --global user.name "${APP_SLUG}[bot]" + git config --global user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com" + + - name: Run the batch + env: + BACKPORT_BATCH: ${{ runner.temp }}/backport-batch + BACKPORT_BATCH_TOKEN: ${{ steps.app-token.outputs.token }} + BACKPORT_BATCH_BOT: ${{ steps.app-token.outputs.app-slug }}[bot] + BACKPORT_ALERT_TEAM: ${{ vars.BACKPORT_ALERT_TEAM }} + run: hack/backport-batch-scheduled.sh --no-workflow-files diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml deleted file mode 100644 index a2b9b75a9..000000000 --- a/.github/workflows/backport.yaml +++ /dev/null @@ -1,305 +0,0 @@ -name: Automatic Backport - -on: - pull_request_target: - types: [closed, labeled] # fires when PR is closed (merged) or labeled - -# Two mechanisms here, answering two different failures. -# -# A run joins its concurrency group before any job-level `if` is evaluated, so -# `prepare`'s guard below cannot stop a label event from disturbing the run -# already backporting this PR; it only decides what the label run does after the -# damage. Labels set with the default GITHUB_TOKEN start no run, but labels set -# by third-party GitHub Apps do, and they arrive in bursts. Under a plain -# `cancel-in-progress: true` each one cancelled the backport the merge had just -# started, and nothing reports that beyond a cancelled run sitting next to a -# green one. -# -# The delivery survived anyway, because the old guard admitted a run on the PR's -# cumulative labels: the last label of a burst requalified and redid the work. -# `prepare`'s narrowing below removes that accidental safety net, so the two -# changes land together — narrowing alone would turn a noisy delivery into an -# absent one. -# -# So the group key moves a label event that is NOT a backport request aside, and -# `cancel-in-progress` stops one that IS from cancelling rather than queuing. -# Both are load-bearing. `backport` and `backport-previous` are separate requests -# that both have to complete, so they share the main group and must not cancel -# each other; every other label does no work here and only has to stay out of -# the way. -# -# That is also why this file does not simply copy pull-requests.yaml, which -# splits the key and leaves cancellation unconditional: a discarded label run -# there publishes nothing, so isolating it is the whole fix. -# -# Queuing protects a pending run from the run ahead of it and never from the one -# behind: the group holds a single pending run, and a newer one evicts it. With -# the split in place that costs nothing, and for the same reason the fan-out -# below is harmless — the only events left in this group are backport requests, -# the evictor is by definition the newer of the two, and `prepare`'s `Check which -# labels are present` step reads the cumulative label set. So the survivor covers -# a superset of what the evicted run would have delivered, for any sequence that -# only adds labels. Removing one in between is the exception and needs no cover: -# the evictor then carries less because the request was withdrawn. -# -# That is exactly what the split buys. Without it the evictor could be any label -# at all, and a run that does not qualify delivers nothing in place of the -# request it displaced. -# -# `queue: max` raises the single pending slot to a hundred and is the obvious -# thing to reach for instead of the split. Taking it means giving up the -# conditional cancel, because `queue: max` and `cancel-in-progress: true` are a -# validation error together and this key still evaluates to true on a merge. -# The trade is available and was not taken: eviction is what keeps a burst to one -# surviving run rather than a hundred queued ones, and the feature only shipped -# 2026-05-07, on a path that delivers fixes to release lines. -concurrency: - group: backport-${{ github.workflow }}-${{ github.event.pull_request.number }}${{ (github.event.action == 'labeled' && github.event.label.name != 'backport' && github.event.label.name != 'backport-previous' && github.event.label.name != 'kind/backport' && github.event.label.name != 'kind/backport-previous') && '-label' || '' }} - # Stated positively, matching the group key above. `!= 'labeled'` is equivalent - # only while `types:` holds exactly these two, and that form fails open: adding - # `unlabeled` or `reopened` there would hand those events cancellation in the - # main group, which is the behaviour this file exists to remove. - # - # The positive form removes the cancelling, not the occupying. A third type - # still lands in the main group, still takes its single pending slot, and its - # `prepare` still skips — so it can evict a genuine backport request and - # deliver nothing in its place. `types:` is load-bearing for that reason and is - # pinned in hack/release-freeze-contract.bats; widening it needs the group key - # widened with it. - cancel-in-progress: ${{ github.event.action == 'closed' }} - -# Top-level token defaults to read-only. A job-level block replaces this one -# rather than adding to it, so each job restates every scope it needs. -permissions: - contents: read - -jobs: - # Determine which backports are needed - prepare: - # This guard admits each action on the labels it is entitled to. `closed` - # carries no label of its own, so it reads the set the PR ended up with. - # `labeled` reads only the label just added: matching the cumulative set - # here means every later unrelated label on a merged PR that still carries - # `backport` re-enters this job to redo a backport already delivered. - # - # Scope, so this is not read as covering the whole job: the `Check which - # labels are present` step below still reads the cumulative set, so a - # `backport-previous` event on a PR also carrying `backport` fans out to - # both matrix legs. That is deliberate, and each state a re-entered leg can - # find is already accounted for: delivered is skipped by the merged-backport - # guard, still open is skipped by backport-action on the 422, and closed - # unmerged gets a fresh backport PR, which is the retry that guard - # deliberately preserves rather than an accident. What the narrowing here - # decides is whether the job RUNS, not which targets it picks. - if: | - github.event.pull_request.merged == true && - github.event.pull_request.base.ref == 'main' && - ( - (github.event.action == 'closed' && (contains(github.event.pull_request.labels.*.name, 'backport') || contains(github.event.pull_request.labels.*.name, 'backport-previous') || contains(github.event.pull_request.labels.*.name, 'kind/backport') || contains(github.event.pull_request.labels.*.name, 'kind/backport-previous'))) || - (github.event.action == 'labeled' && (github.event.label.name == 'backport' || github.event.label.name == 'backport-previous' || github.event.label.name == 'kind/backport' || github.event.label.name == 'kind/backport-previous')) - ) - # GitHub API calls only — no docker, no repo toolchain: GitHub-hosted. - runs-on: ubuntu-latest - # Queuing gives a stuck run a way to hurt the next one that a cancelling key - # did not: the label event that used to kill it now waits behind it rather - # than clearing it. - # - # This ceiling bounds a stuck JOB, which is the case it can reach. A run can - # also wedge at the RUN level with every job already finished, and no - # job-level key expires that one; it holds the group until GitHub retires the - # run. Rare but real: run 31232645133 has been `in_progress` since - # 2026-08-08 with `prepare` and `backport` both skipped. Nothing here - # reaches that state, so docs/release.md tells an operator to look for a run - # still in flight and clear it before re-applying the label. - # - # What it bounds is EXECUTION, not the wait for a runner. GitHub's limits - # page says "job execution time" for the 6-hour job cap while spelling out - # that the 35-day run cap "includes execution duration, and time spent on - # waiting and approval" — the contrast is deliberate — and - # orgs/community#50926 shows a job sitting six hours on "Waiting for a runner - # to pick up this job" under `timeout-minutes: 5`. So queue time is a third - # unbounded case alongside the run-level wedge, not something this covers. - # - # 30 is therefore generous rather than calculated. Execution on run - # 29844315422 was 6s for `prepare` and 2s and 16s for the two matrix legs, - # against 9m24s, 5m28s and 5m45s of queue for the same three jobs. Every - # execution measured on this workflow has been seconds, and no maximum is - # quoted on purpose: an absolute maximum over a growing set is stale as soon - # as a slower run lands. The value leaves room for a cherry-pick far larger - # than anything seen while replacing the 6-hour default, and nothing here is - # sized against the queue because nothing here can be. - timeout-minutes: 30 - permissions: - contents: read - outputs: - backport_current: ${{ steps.labels.outputs.backport }} - backport_previous: ${{ steps.labels.outputs.backport_previous }} - current_branch: ${{ steps.branches.outputs.current_branch }} - previous_branch: ${{ steps.branches.outputs.previous_branch }} - steps: - - name: Check which labels are present - id: labels - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - const pr = context.payload.pull_request; - const labels = pr.labels.map(l => l.name); - // TRANSITIONAL: both spellings are accepted so the rename can land - // in any order relative to the org-level dosubot, which still - // applies the legacy names and is today the dominant applier on - // real backports. Once dosubot's PR labelling is switched off and - // the in-repo labeler is the only applier, the legacy arms here, - // in the `if:` gate above and in the concurrency group can all be - // deleted in one pass. - const isBackport = labels.includes('kind/backport') || labels.includes('backport'); - const isBackportPrevious = labels.includes('kind/backport-previous') || labels.includes('backport-previous'); - - core.setOutput('backport', isBackport ? 'true' : 'false'); - core.setOutput('backport_previous', isBackportPrevious ? 'true' : 'false'); - - console.log(`backport label: ${isBackport}, backport-previous label: ${isBackportPrevious}`); - - - name: Determine target branches - id: branches - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - // Enumerate the real release-X.Y branches rather than deriving them - // from the latest published release. - // - // Since cutting an rc freezes the line (cut-prerelease.yaml), - // release-X.Y is created when the first vX.Y.0-rc.N is cut — before - // vX.Y.0 is ever published. getLatestRelease returns the newest - // published STABLE, so throughout the freeze window it still named - // the previous line: a `backport` on a fix for the release being - // stabilised would have landed on release-X.(Y-1) and missed the - // release it was written for. That window is exactly when backports - // matter most, since the frozen branch is the only way in. - // - // Enumerating branches also drops the contiguous-minor assumption: - // `previous` used to be computed as min-1, which named a - // non-existent branch whenever a minor was skipped. - const branches = await github.paginate(github.rest.repos.listBranches, { - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 100 - }); - - const lines = branches - .map(b => /^release-(\d+)\.(\d+)$/.exec(b.name)) - .filter(Boolean) - .map(m => ({ name: m[0], maj: Number(m[1]), min: Number(m[2]) })) - .sort((a, b) => (b.maj - a.maj) || (b.min - a.min)); - - if (lines.length === 0) { - core.setFailed('No release-X.Y branches found; cannot determine backport target.'); - return; - } - - const currentBranch = lines[0].name; - const previousBranch = lines.length > 1 ? lines[1].name : ''; - - core.setOutput('current_branch', currentBranch); - core.setOutput('previous_branch', previousBranch); - - console.log(`Release lines found: ${lines.map(l => l.name).join(', ')}`); - console.log(`Current branch: ${currentBranch}, Previous branch: ${previousBranch || 'N/A'}`); - - // Both targets come from the enumeration above, so they exist by - // construction — no getBranch probe needed. What CAN still fail is - // asking for a previous line when only one exists. - if (!previousBranch && '${{ steps.labels.outputs.backport_previous }}' === 'true') { - core.setFailed('backport-previous requested, but only one release line exists.'); - return; - } - - backport: - needs: prepare - if: | - github.event.pull_request.merged == true && - github.event.pull_request.base.ref == 'main' && - (needs.prepare.outputs.backport_current == 'true' || needs.prepare.outputs.backport_previous == 'true') - # git cherry-picks via backport-action — no docker, no repo toolchain: GitHub-hosted. - runs-on: ubuntu-latest - # See `prepare` above: a queued request waits on this one, so it needs a - # ceiling rather than the 6-hour default. Bounds this job's execution only; - # the wait for a runner is outside it either way. - timeout-minutes: 30 - permissions: - contents: write - pull-requests: write - strategy: - matrix: - backport_type: [current, previous] - steps: - # 1. Determine target branch based on matrix - - name: Set target branch - id: target - if: | - (matrix.backport_type == 'current' && needs.prepare.outputs.backport_current == 'true') || - (matrix.backport_type == 'previous' && needs.prepare.outputs.backport_previous == 'true') - run: | - if [ "${{ matrix.backport_type }}" == "current" ]; then - echo "branch=${{ needs.prepare.outputs.current_branch }}" >> $GITHUB_OUTPUT - echo "Target branch: ${{ needs.prepare.outputs.current_branch }}" - else - echo "branch=${{ needs.prepare.outputs.previous_branch }}" >> $GITHUB_OUTPUT - echo "Target branch: ${{ needs.prepare.outputs.previous_branch }}" - fi - - # 2. Skip if this target already has a merged backport. - # - # The action's backport branch name is deterministic, - # backport--to-. A merged PR with that head - # means an earlier run already delivered this change to the target — a - # second label event (e.g. backport-previous added after backport - # already merged) would re-enter this job and cherry-pick commits the - # target branch already has. Git reports that as an empty pick (exit 1) - # rather than a conflict, which draft_commit_conflicts below misreads - # as a real conflict and reports as a failed backport on the original - # PR. Skipping here avoids the false report; it does not touch the - # empty-pick behavior itself, which lives in the action. - - name: Check for existing merged backport - id: guard - if: steps.target.outcome == 'success' - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - const targetBranch = '${{ steps.target.outputs.branch }}'; - const head = `backport-${context.payload.pull_request.number}-to-${targetBranch}`; - const { data: prs } = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'closed', - head: `${context.repo.owner}:${head}`, - }); - const alreadyMerged = prs.some(p => p.merged_at !== null); - core.setOutput('already_merged', alreadyMerged ? 'true' : 'false'); - if (alreadyMerged) { - console.log(`Skipping backport to ${targetBranch}: ${head} already merged.`); - } - - # 3. Checkout (required by backport‑action) - - name: Checkout repository - if: steps.target.outcome == 'success' && steps.guard.outputs.already_merged != 'true' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - # 4. Create the back‑port pull request - - name: Create back‑port PR - id: backport - if: steps.target.outcome == 'success' && steps.guard.outputs.already_merged != 'true' - uses: korthout/backport-action@0193454f0c5947491d348f33a275c119f30eb736 # v3.2.1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - label_pattern: '' # don't read labels for targets - target_branches: ${{ steps.target.outputs.branch }} - merge_commits: skip - # conflict_resolution lives inside the experimental JSON input, not as a - # top-level key. Passed top-level it is silently ignored and the action - # falls back to the experimental default ("fail"), which on a conflict - # creates no PR and only comments. Nesting it here makes a conflicting - # backport open a draft PR with the conflict committed for manual fixup. - experimental: | - { - "conflict_resolution": "draft_commit_conflicts" - } diff --git a/.github/workflows/changelog-generate.yaml b/.github/workflows/changelog-generate.yaml index 10452fb85..ff67b4f7e 100644 --- a/.github/workflows/changelog-generate.yaml +++ b/.github/workflows/changelog-generate.yaml @@ -154,6 +154,7 @@ jobs: sparse-checkout: | hack docs/agents + docs/release persist-credentials: false # The agent reads docs/agents/changelog.md from the WORKING TREE, which is @@ -163,18 +164,17 @@ jobs: # detached rc-tag checkout is exactly that, so a COMPLIANT agent would # silently decline to generate on every promotion of an existing rc. # Overlay the current instructions so the agent reads today's contract. - # changelog.md sends the agent to the checks in docs/release.md and to - # hack/validate-changelog.sh, so those are overlaid too: the rc tag's - # copies predate any check added since. docs/release.md reaches - # .release-tooling only because the sparse checkout above is in cone - # mode (the action's default), which carries the files of every parent - # directory of docs/agents. + # changelog.md sends the agent to the checks in docs/release/changelogs.md + # and to hack/validate-changelog.sh, so those are overlaid too: the rc + # tag's copies predate any check added since, and an rc cut before the + # release docs were split into docs/release/ has no such directory. - name: Refresh agent instructions run: | cp .release-tooling/docs/agents/changelog.md docs/agents/changelog.md - cp .release-tooling/docs/release.md docs/release.md + mkdir -p docs/release + cp .release-tooling/docs/release/changelogs.md docs/release/changelogs.md cp .release-tooling/hack/validate-changelog.sh hack/validate-changelog.sh - echo "Refreshed docs/agents/changelog.md, docs/release.md and hack/validate-changelog.sh from ${GITHUB_REF_NAME}." + echo "Refreshed docs/agents/changelog.md, docs/release/changelogs.md and hack/validate-changelog.sh from ${GITHUB_REF_NAME}." # Skip-priority + validate-then-fall-through: use a pre-existing changelog # only if it is USABLE, so an invalid higher-priority source can never diff --git a/.github/workflows/cut-prerelease.yaml b/.github/workflows/cut-prerelease.yaml index e740c2978..bfd159d94 100644 --- a/.github/workflows/cut-prerelease.yaml +++ b/.github/workflows/cut-prerelease.yaml @@ -6,7 +6,7 @@ name: Cut Pre-release Tag # This is the single entry point for creating a pre-release. Stable vX.Y.Z tags # are never cut here (or by hand) — they are created write-once by # pull-requests-release.yaml finalize at a promote PR's merge commit. See -# docs/release.md and #2677. +# docs/release/stable-tags.md and #2677. # # Dispatch from the branch the tag belongs to: `main` for a new minor's # pre-release (vX.Y.0-rc.N), or `release-X.Y` for a patch line (vX.Y.Z-rc.N). diff --git a/.github/workflows/e2e-fork.yaml b/.github/workflows/e2e-fork.yaml index df6ed1268..f988632e9 100644 --- a/.github/workflows/e2e-fork.yaml +++ b/.github/workflows/e2e-fork.yaml @@ -771,21 +771,6 @@ jobs: name: cozyreport path: _out/cozyreport.tgz if-no-files-found: warn - # Both halves, and in this order: the compose project owns the nodes and - # the zpools outlive it. - - name: Tear down sandbox - if: always() - run: | - # Annotated, not fatal: same reasoning as the in-tree lane. The runner - # is discarded after the job, so a dirty host has no next tenant, and - # failing here would post the required "E2E Tests" status as a failure - # indistinguishable from a real one. - cleanup_rc=0 - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$? - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true - if [ "$cleanup_rc" -ne 0 ]; then - echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool" - fi # Flip the pending "E2E Tests" commit status posted by `resolve` to its # terminal state. Runs on failure too, so a failed publish or e2e turns the diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml index a241b8169..7aca218f7 100644 --- a/.github/workflows/pr-labeler.yaml +++ b/.github/workflows/pr-labeler.yaml @@ -24,7 +24,7 @@ jobs: with: script: | // Conventional Commits types accepted by Cozystack (per docs/agents/contributing.md): - // feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert + // feat, fix, security, docs, style, refactor, perf, test, build, ci, chore, revert // Mapping below maps a subset to kind/* — types not listed do not produce a kind/*. const typeToKind = { feat: 'kind/feature', @@ -32,7 +32,7 @@ jobs: docs: 'kind/documentation', chore: 'kind/cleanup', refactor: 'kind/cleanup', - // style, perf, test, build, ci, revert — no kind mapping + // security, style, perf, test, build, ci, revert — no kind mapping }; // scope -> area/* mapping. Keys are the scopes observed in cozystack issues @@ -296,3 +296,82 @@ jobs: labels: newLabels, }); core.info(`Applied labels: ${newLabels.join(', ')}`); + + # Backports are automatic (docs/release/backports.md), so a pull request + # against a release branch from outside the team is closed with a pointer to + # the policy. Only when it is opened: a maintainer keeps a fix that exists + # only on a release line by reopening it. + # + # pull_request_target hands a write token to a fork's pull request, so this + # job never touches the head: actions/checkout takes the event's own ref, + # which is the base repository, the script reads only the event JSON and the + # author's permission on the repository, and no `${{ }}` expression reaches + # the shell. The permission read needs repository metadata read access, which + # a GITHUB_TOKEN always carries and `permissions:` has no key for. + close-manual-backport: + if: >- + github.event.action == 'opened' && + startsWith(github.event.pull_request.base.ref, 'release-') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read # check out hack/ from the base repository + pull-requests: write # close the pull request with a comment + steps: + - name: Check out the base repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + sparse-checkout: hack + - name: Close a backport opened by hand from outside the team + env: + GH_TOKEN: ${{ github.token }} + run: hack/close-manual-backport.sh "$GITHUB_EVENT_PATH" + + # A batch pull request carries each original in the shape cmd/backport-batch + # gives it. fold checks one resolution, and only when whoever resolved the + # conflict runs it; this checks every original on the batch on each push. + # + # pull_request_target takes the workflow from main for a pull request into + # any branch, so the check reaches release-* lines, which do not carry it; a + # pull_request job would run the line's own workflows. The tool is built from + # main, since the lines do not carry it either. Nothing from the pull request + # is checked out or run: its head is fetched as data and read with git, and + # no `${{ }}` expression reaches the shell. Only a branch of this repository + # named like a batch is checked. + backport-batch-verify: + name: Backport batch shape + if: >- + startsWith(github.event.pull_request.base.ref, 'release-') && + startsWith(github.event.pull_request.head.ref, 'backport-batch/') && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + # The pipe falls back to direct on any proxy error (hack/common-envs.mk). + GOPROXY: "https://proxy.golang.org|direct" + steps: + - name: Check out main + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: main + fetch-depth: 0 + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: false + - name: Build backport-batch from main + run: go build -o "$RUNNER_TEMP/backport-batch" ./cmd/backport-batch + - name: Fetch the batch head as data + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: git fetch --no-tags --quiet origin "$HEAD_SHA" + - name: Verify every original on the batch + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: '"$RUNNER_TEMP/backport-batch" verify --line "$BASE_REF" --head "$HEAD_SHA"' diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index dbec873fb..c2e98276d 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -89,16 +89,14 @@ jobs: runs-on: ubuntu-22.04 permissions: contents: read - # A backport PR is the one shape whose author cannot fix a finding: the - # backport action cherry-picks merged commits verbatim, so their messages + # A backport batch is the one shape whose author cannot fix a finding: + # cmd/backport-batch cherry-picks merged commits verbatim, so their messages # are history, and roughly a quarter of the commits on main predate the - # rule. Matched on the branch name the action generates, - # backport--to-, rather than on the prefix alone; the label that - # asks for a backport sits on the original PR, not on the generated one, so - # a branch name is the only signal a pull_request event carries. That makes - # the exemption a convention rather than something enforced: anyone can name - # a branch this way. Every other PR, fork included, is checked. - if: ${{ !(startsWith(github.head_ref, 'backport-') && contains(github.head_ref, '-to-')) }} + # rule. Matched on the branch name the batch uses, + # backport-batch/-, and only from a branch of this repository, + # which takes write access to push: a fork can name a branch the same way. + # Every other PR, fork included, is checked. + if: ${{ !(startsWith(github.head_ref, 'backport-batch/') && github.event.pull_request.head.repo.full_name == github.repository) }} steps: - name: Checkout head uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 diff --git a/.github/workflows/promote-rc.yaml b/.github/workflows/promote-rc.yaml index fdb57e9e3..d0567018a 100644 --- a/.github/workflows/promote-rc.yaml +++ b/.github/workflows/promote-rc.yaml @@ -327,9 +327,12 @@ jobs: // A maintenance branch created before the candidate pipeline landed // would execute its old pull_request workflows after merge. Refuse // before the first registry write unless the actual target base has - // both trusted verification gates and their helpers. Backport this - // change to an existing release-X.Y line before promoting its next - // patch; the rc-derived head does not need to carry the tooling. + // both trusted verification gates and their helpers. An existing + // release-X.Y line gets them before its next patch is promoted, + // through the backport batch with backport/include on the pull + // request that brought them to main, or through a maintainer's pull + // request against the line; the rc-derived head does not need to + // carry the tooling. // // Every file the base-checked-out verifier reaches for belongs in // the list below, its own sourced libraries included: those are @@ -363,7 +366,7 @@ jobs: }); } catch (e) { if (e.status === 404) { - core.setFailed(`Target base '${baseBranch}' lacks '${path}'. Backport the candidate-aware promotion pipeline before dispatch.`); + core.setFailed(`Target base '${baseBranch}' lacks '${path}'. Bring the candidate-aware promotion pipeline to it before dispatch: backport/include on the pull request that added it to main, or a maintainer pull request against the line.`); return; } throw e; @@ -374,7 +377,7 @@ jobs: } const content = Buffer.from(file.data.content, 'base64').toString('utf8'); if (!content.includes(marker)) { - core.setFailed(`Target base '${baseBranch}' has an old '${path}'. Backport the candidate-aware promotion pipeline before dispatch.`); + core.setFailed(`Target base '${baseBranch}' has an old '${path}'. Bring the candidate-aware promotion pipeline to it before dispatch: backport/include on the pull request that added it to main, or a maintainer pull request against the line.`); return; } } @@ -603,10 +606,11 @@ jobs: tar -xzf "$syft_dir/syft.tar.gz" -C "$syft_dir" syft rm -f "$syft_dir/syft.tar.gz" echo "$syft_dir" >> "$GITHUB_PATH" - # cosign is the version docs/release.md asks users for, so the job checks - # its own signature the way they will. The action verifies that binary - # against cosign's keyless and KMS bundles with its digest-pinned - # bootstrap cosign. Bump with tags.yaml; the contract test holds them equal. + # cosign is the version docs/release/signatures-and-sbom.md asks users + # for, so the job checks its own signature the way they will. The action + # verifies that binary against cosign's keyless and KMS bundles with its + # digest-pinned bootstrap cosign. Bump with tags.yaml; the contract test + # holds them equal. - name: Set up cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 with: @@ -1044,9 +1048,10 @@ jobs: PR_STATE="$(gh pr view "${STABLE_BRANCH}" --json state --jq .state 2>/dev/null || echo "")" if [ "${PR_STATE}" = "OPEN" ]; then # Refresh only while the body is still the one this workflow wrote. - # docs/release.md tells reviewers to paste upstream breaking-change - # notes into this body before merge; a re-dispatch that overwrites it - # would destroy that with no trace and no way to recover it. + # docs/release/pre-release-checklist.md tells reviewers to paste + # upstream breaking-change notes into this body before merge; a + # re-dispatch that overwrites it would destroy that with no trace + # and no way to recover it. CURRENT="$(gh pr view "${STABLE_BRANCH}" --json body --jq .body 2>/dev/null || echo "")" if [ "${CURRENT}" = "${BODY}" ]; then echo "PR already open for ${STABLE_BRANCH} with an identical body; nothing to do." diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index b8b06fedf..7d3787ff9 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -1348,28 +1348,6 @@ jobs: Resume from inside: `breakpoint resume`. Otherwise the breakpoint exits 10 minutes after the last SSH session disconnects. - # ▸ Tear down environment (always runs) - # - # Both halves, and in this order: the compose project owns the nodes and - # the zpools outlive it. A leftover pool makes the next run on a reused - # host adopt storage that already carries a dead cluster's volumes. - - name: Tear down sandbox - if: always() - run: | - # Loud, but never the job's verdict. A failed teardown means a dirty - # host, and this runner is discarded after the job, so there is no next - # run to protect -- while failing the job here posts the required - # "E2E Tests" status as a failure that no reader can tell apart from a - # real one. The annotation is what a reused host (a developer machine) - # needs; `delete-cluster-container` returning non-zero is not evidence - # about the product. - cleanup_rc=0 - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$? - make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true - if [ "$cleanup_rc" -ne 0 ]; then - echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool" - fi - # Report the required "E2E Tests" gate as a commit status rather than relying # on a job named "E2E Tests". A required *job* that is skipped counts # as PASSED by branch protection — desired for docs-only PRs, but the exact diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 96e495abc..d8b40aadc 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -39,8 +39,7 @@ jobs: security/triage-needed,security/in-progress, triage/accepted,kind/regression exempt-pr-labels: >- - lifecycle/frozen,do-not-merge/hold, - kind/backport,kind/backport-previous + lifecycle/frozen,do-not-merge/hold days-before-stale: 60 days-before-close: 14 operations-per-run: 100 diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 8c123fdb1..f35ca8265 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -556,8 +556,8 @@ jobs: tar -xzf "$syft_dir/syft.tar.gz" -C "$syft_dir" syft rm -f "$syft_dir/syft.tar.gz" echo "$syft_dir" >> "$GITHUB_PATH" - # cosign is the version docs/release.md asks users for, as in - # promote-rc.yaml. Bump with it. + # cosign is the version docs/release/signatures-and-sbom.md asks users + # for, as in promote-rc.yaml. Bump with it. - name: Set up cosign if: needs.build-amd64.outputs.release_exists == 'false' uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 diff --git a/.gitignore b/.gitignore index db787248d..f1b99efdb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ _repos # root-level binary waiting to happen — each ~80 MB, and nothing else here # matched them, leaving `git add .` one keystroke from committing one. /api-gate -/backport-audit +/backport-batch /backup-controller /backupstrategy-controller /check-readiness @@ -23,6 +23,7 @@ _repos /flux-shard-operator /kubeovn-plunger /lineage-controller-webhook +/observability-proxy /securitygroup-controller # User-specific stuff diff --git a/AGENTS.md b/AGENTS.md index c51ab06cf..f35097317 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,11 @@ This file provides structured guidance for AI coding assistants and agents worki - **Release creation** (e.g., "create release", "prepare release", "tag release", "make a release") - Read: [`releasing.md`](./docs/agents/releasing.md) - - Action: Read the file and follow the referenced release process in `docs/release.md` + - Action: Read the file and follow the referenced release process: [`docs/release.md`](./docs/release.md) indexes one page per topic under `docs/release/` + +- **Backports, patch content** (e.g., "backport this fix", "will this reach 1.6", "what goes into the next patch", "open a PR against release-X.Y") + - Read: [`backports.md`](./docs/release/backports.md) + - Action: Read the entire file. Backports are automatic: a fix merged to `main` with a `fix` title, or a `security` one for a fix that comes through a security advisory, reaches the release lines the batch runs for, the newest released lines and any line in its rc window, so never open a backport PR by hand. A pull request against `release-*` from outside the team is closed automatically when it is opened; a maintainer keeps a fix with no counterpart on `main` by reopening it - **Image references, tags and digests** (e.g., "why is this image tagged rc", "add an image to a package", "change how an image is stamped", "promotion/nightly missed an image", "where do image refs live") - Read: [`image-refs.md`](./docs/agents/image-refs.md) diff --git a/GENERAL_TECHNICAL_REVIEW.md b/GENERAL_TECHNICAL_REVIEW.md index 9e7818b55..81b83fb8a 100644 --- a/GENERAL_TECHNICAL_REVIEW.md +++ b/GENERAL_TECHNICAL_REVIEW.md @@ -88,7 +88,7 @@ ### Rollout, Upgrade and Rollback Planning -* **Compatibility with Kubernetes & frequency.** Cozystack ships a curated, tested Kubernetes version per release; tenant clusters are Certified Kubernetes following upstream skew/support. Platform upgrades are delivered as new Cozystack releases. The authoritative list of what has shipped is the [GitHub Releases page](https://github.com/cozystack/cozystack/releases); which release lines are still receiving security fixes is defined as a rolling rule in [SECURITY.md](https://github.com/cozystack/cozystack/blob/main/SECURITY.md): the latest stable minor and the two preceding minors are supported, older `1.x` minors receive critical and upgrade-blocking fixes at maintainer discretion, `v0.41.x` is end of life and anything earlier is unsupported. Specific version numbers are deliberately not restated here, because they drift with every release. Note that being in a supported line does not entail receiving every fix: the project has no documented rule for which supported lines a given fix is backported to, and that is recorded in `SECURITY.md` rather than presented as a policy. +* **Compatibility with Kubernetes & frequency.** Cozystack ships a curated, tested Kubernetes version per release; tenant clusters are Certified Kubernetes following upstream skew/support. Platform upgrades are delivered as new Cozystack releases. The authoritative list of what has shipped is the [GitHub Releases page](https://github.com/cozystack/cozystack/releases); which release lines are still receiving security fixes is defined as a rolling rule in [SECURITY.md](https://github.com/cozystack/cozystack/blob/main/SECURITY.md): the two latest minor releases are supported, older `1.x` minors receive critical and upgrade-blocking fixes at maintainer discretion, `v0.41.x` is end of life and anything earlier is unsupported. Specific version numbers are deliberately not restated here, because they drift with every release. Note that being in a supported line does not entail receiving every fix: which fixes reach a supported line is the [backport policy](https://github.com/cozystack/cozystack/blob/main/docs/release/backports.md), which keeps breaking changes, platform migrations and the like for the next minor. * **Upgrade procedure** ([upgrade docs](https://cozystack.io/docs/v1.6/operations/cluster/upgrade/)): run the troubleshooting/health checklist; annotate the `cozy-system` namespace and `cozystack-version` ConfigMap with `helm.sh/resource-policy=keep`; `helm upgrade cozystack oci://ghcr.io/cozystack/cozystack/cozy-installer --version X.Y.Z` (**do not** use `--reuse-values` — the chart pins OCI repo/package versions); verify pods and `kubectl get hr -A`. * **Rollback.** GitOps/Helm-managed with versioned releases; roll back by targeting the previous release (previous HelmRelease revisions) via the same mechanism. Stateful data (databases, VM disks) persists on replicated storage across control-plane rollbacks. **No dedicated rollback runbook is published**, and version-skip constraints for downgrades are not documented; the [upgrade docs](https://cozystack.io/docs/v1.6/operations/cluster/upgrade/) currently cover the forward path only. * **How a rollout/rollback can fail & impact.** Risks: incompatible CRD schema change, storage/CNI component upgrades requiring node reboots (on the Talos path), or a database-operator major bump. Impact is scoped to the component; running tenant workloads generally continue, though storage/CNI upgrades can transiently affect networking or volume attachment during node rolls. `[maintainers: confirm whether upgrades trigger Talos node reboots]` diff --git a/Makefile b/Makefile index 6da0e1037..2feea00e9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check bats-posix-compat-tests print-bats-unit-files print-bats-posix-compat-files print-bats-jobs rd-presets-check migrations-target-check test test-controllers test-controllers-envtest test-backport-audit preflight +.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check bats-posix-compat-tests print-bats-unit-files print-bats-posix-compat-files print-bats-jobs rd-presets-check migrations-target-check test test-controllers test-controllers-envtest test-backport-batch preflight include hack/common-envs.mk @@ -17,6 +17,7 @@ build: build-deps make -C packages/system/cozystack-api image make -C packages/system/cozystack-controller image make -C packages/system/backup-controller image + make -C packages/system/observability-proxy image make -C packages/system/backupstrategy-controller image make -C packages/system/lineage-controller-webhook image make -C packages/system/migration-controller image @@ -119,7 +120,7 @@ test: make -C packages/core/testing apply make -C packages/core/testing e2e -unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-audit migrations-target-check +unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check helm-unit-tests: hack/helm-unit-tests.sh @@ -194,9 +195,9 @@ test-controllers-envtest: $(SETUP_ENVTEST) test-check-readiness: go test ./test/check-readiness/ -count=1 -# ./cmd/... is excluded from go-unit-tests, so the audit needs its own target. -test-backport-audit: - go test ./cmd/backport-audit/ -count=1 +# ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target. +test-backport-batch: + go test ./cmd/backport-batch/... -count=1 # Unit discovery is one level deep. Live-cluster files need an invocation in # packages/core/testing; hack/bats-runner-coverage.bats checks that boundary. @@ -294,7 +295,7 @@ bats-posix-compat-tests: # own too; hack/unit-lane-no-cluster.bats fails when one is missing. NO_CLUSTER_TARGETS := unit-tests test-controllers helm-unit-tests bats-unit-tests \ bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check \ - test-check-readiness test-backport-audit migrations-target-check + test-check-readiness test-backport-batch migrations-target-check $(NO_CLUSTER_TARGETS): export KUBECONFIG := /dev/null $(NO_CLUSTER_TARGETS): export KUBERNETES_SERVICE_HOST := diff --git a/SECURITY.md b/SECURITY.md index 8db8eb742..552ed1950 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ Cozystack integrates and ships many upstream cloud native components. If you bel ## Supported Versions -The Cozystack project maintains three release lines at a time: **the latest stable minor and the two minors preceding it.** That is a rolling rule rather than a list of version numbers, so it does not go stale between releases — read it against the GitHub Releases page to see which concrete lines it currently names: +The Cozystack project supports two release lines at a time: **the two latest minor releases**, the latest stable minor and the one before it. When a new minor ships its first stable release it becomes the latest, and the older of the two drops out. That is a rolling rule rather than a list of version numbers, so it does not go stale between releases — read it against the GitHub Releases page to see which concrete lines it currently names: @@ -16,16 +16,16 @@ Note that the rule is keyed to release lines, not to the existence of a `release | Version line | Status | Notes | | --- | --- | --- | -| Latest stable minor | Supported | Current stable release line. Security fixes land here first, and patch releases are cut from it. | -| The two minors preceding the latest | Supported | Receive security and important maintenance fixes, backported from the current line. | +| Latest stable minor | Supported | Current stable release line, the first supported line a fix reaches. Patch releases are cut from it. | +| The minor before the latest | Supported | Receives the same security and maintenance fixes, backported once the latest line carries them. | | Older `1.x` minors | Limited support | Critical and upgrade-blocking fixes may be backported at maintainer discretion. Adopters are encouraged to move to a supported line. | | `v0.41.x` | End of life | The final pre-v1 line. It received patch releases through March 2026 during the v0 to v1 transition and no longer does; no further fixes are planned. | | Everything before `v0.41` | Not supported | Long superseded. Upgrade to a `1.x` line. | | `alpha`, `beta`, `rc` releases | Not supported | Pre-release builds are for testing and evaluation only. | -Being in a supported row does not mean a line receives every fix. **The project does not have a documented rule for choosing which supported lines a given fix is backported to** — that decision is taken per fix by the maintainers, weighing severity against the risk and effort of the backport. This is stated rather than dressed up as a policy because no such policy exists; adopters who need a specific fix on a specific line should ask. +Being in a supported row does not mean a line receives every fix. Fixes reach the supported lines through the backport batch, by the rules in [`docs/release/backports.md`](docs/release/backports.md): a fix merged into `main` with a `fix` or `security` title goes onto both lines, unless the policy keeps it for the next minor, a breaking change or a platform migration for instance, and a maintainer can override that for a single fix. Adopters who need a specific fix on a specific line should ask. -Reporting a vulnerability against a line with limited support or at end of life is still welcome. We will confirm whether the issue also affects a supported line and fix it there; whether the fix is additionally backported to the older line is a per-fix maintainer decision on the same basis. +Reporting a vulnerability against a line with limited support or at end of life is still welcome. We will confirm whether the issue also affects a supported line and fix it there; whether the fix is additionally backported to the older line is a per-fix maintainer decision. ## Reporting a Vulnerability diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index f0415f398..4c0d80ab8 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -58,7 +58,7 @@ type ConfigSpec struct { // Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret. // +kubebuilder:default:={} Users map[string]User `json:"users,omitempty"` - // Databases configuration map. + // Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. // +kubebuilder:default:={} Databases map[string]Database `json:"databases,omitempty"` // Backup configuration. diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go index 509c1e6ce..c2363749c 100644 --- a/api/apps/v1alpha1/redis/types.go +++ b/api/apps/v1alpha1/redis/types.go @@ -17,7 +17,7 @@ type Config struct { } type ConfigSpec struct { - // Number of Redis replicas. + // Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to. // +kubebuilder:default:=2 Replicas int `json:"replicas"` // Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied. @@ -51,7 +51,7 @@ type ConfigSpec struct { // Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. // +kubebuilder:validation:Minimum=0 MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"` - // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. + // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. // +kubebuilder:default:={} Sentinel Sentinel `json:"sentinel,omitempty"` } diff --git a/cmd/backport-audit/README.md b/cmd/backport-audit/README.md deleted file mode 100644 index da446caba..000000000 --- a/cmd/backport-audit/README.md +++ /dev/null @@ -1,226 +0,0 @@ -# backport-audit - -Answers one question, before a release or an rc is cut: did everything labelled for backport actually land on the release branch? It prints the URLs of the ones that did not, and exits non-zero, so it can gate the cut. - -The manual alternative is `gh pr list --search "is:merged label:kind/backport"`, which lists every PR ever labelled and says nothing about whether the change reached the branch. Answering that by hand means reading the branch history per PR, which is where backports quietly go missing. - -```console -$ go run ./cmd/backport-audit release-1.5 - -=== release-1.5 === 43 candidate PRs: MISSING=2 pending=3 dropped=5 backported=33 - - MISSING -- labelled for this line, no trace of it here (2): - https://github.com/cozystack/cozystack/pull/3449 - #3449 ci(release): move release validation to rc time — e2e lane, changelog button, promote gate, docs at promote - label=kind/backport-previous author=myasnikovdaniil merged=2026-07-29 -- no backport PR, nothing on branch -``` - -The URL is always the **original PR on main** — the thing you decide about. The evidence line names the backport PR when one exists. A labelled PR already on the branch is counted in the header and never listed, so silence under the outstanding headings means done. Two more sections, described in [Backports the labels do not account for](#backports-the-labels-do-not-account-for), start from the backport PRs on the branch instead of from labels: `DUPLICATE`, which is outstanding, and `UNLABELLED`, which is informational. - -## Usage - -```bash -go run ./cmd/backport-audit # the two newest release lines -go run ./cmd/backport-audit release-1.5 # one line -go run ./cmd/backport-audit release-1.6 release-1.5 --json -``` - -Requires `git` and an authenticated `gh` on PATH. The audit does not modify GitHub or working-tree files. Unless `--no-fetch` is set, it updates local Git refs and objects while fetching. - -| Flag | Meaning | -|------|---------| -| `--remote NAME` | Git remote holding the release branches (default `origin`) | -| `--limit N` | Max merged PRs scanned per label spelling (default 1000, which is also the ceiling — see [Exit code](#exit-code)) | -| `--no-fetch` | Skip `git fetch`; both main and release refs must be current so shared history stays out of backport evidence | -| `--json` | Machine-readable output | -| `--no-color` | Disable color (auto-disabled on a non-TTY and under `--json`) | - -Arguments and flags may be given in any order. - -## Exit code - -`0` when nothing is outstanding, `1` when something is, `2` when the audit could not be completed. Outstanding means a `MISSING`, `partial`, `unverified`, `pending` or `dropped` verdict, or any `DUPLICATE` entry; `UNLABELLED` entries never move the exit code. That is the point of the tool, so it holds in `--json` mode too: - -```bash -go build ./cmd/backport-audit -./backport-audit release-1.5 -case $? in - 0) echo "clean" ;; - 1) echo "do not cut yet" ;; - *) echo "the audit itself failed; no answer yet" ;; -esac -``` - -Use the built binary (`go build` as above, or `go install ./cmd/backport-audit`) wherever the exit code matters. `go run` does not preserve it: any non-zero exit of the program comes back as `go run`'s own `1`, after an `exit status N` line on stderr, so under `go run` an audit that could not be completed looks exactly like one that found outstanding work. `go run` is fine for reading the report. - -Exit `2` covers the cases where an answer cannot be trusted rather than merely being bad news, and a saturated listing is one of them: `gh` truncates at `--limit` in silence, and a truncated list does not make the audit partial, it makes it wrong — the PRs past the cut are reported nowhere and the exit code says clean. Each of the three listings is checked against its cap and fails instead, naming the cap to raise. The label listings go through GitHub search, which never returns more than 1000 results however far it is paginated, so they are checked against the lower of `--limit` and 1000: a `--limit` above 1000 cannot make a listing cut at 1000 pass for complete. A git read that a verdict depends on is the other case: when one fails, a merge commit missing from the local repository for instance, the audit stops with `2` instead of judging the PR without it. - -## Statuses - -| Status | Meaning | Action | -|--------|---------|--------| -| `in-branch` | The PR merged before the branch was cut, so its change is already there | none, no backport was ever needed | -| `backported` | Every commit of the PR is on the branch, or, for a PR of a single commit, a backport PR merged | none | -| `confirmed` | Would be `partial` or `unverified`, but a maintainer attested the backport complete on the merged backport PR; the report names who and links the comment | none | -| `partial` | Some of the PR's commits are on the branch and some are not, listed with SHA and subject — a merged bot conflict draft, which stops at the first commit that did not apply, or a backport squashed by hand | backport the missing commits, or [attest the backport complete](#attesting-a-backport-complete) | -| `unverified` | A backport PR merged, but none of the PR's several commits is on the branch under its own SHA, `-x` reference or subject — typically a backport squashed or reworded by hand | check it, then attest it complete or backport what is missing | -| `pending` | A backport PR exists and is still open, including the drafts the bot opens with the conflict committed | merge it, or finish the draft | -| `dropped` | A backport PR was closed unmerged, reported with whatever reason someone left on it | none if the reason still holds | -| `MISSING` | No backport PR ever existed and nothing on the branch matches | cherry-pick it by hand, or drop it deliberately | - -`dropped` is usually healthy — a maintainer deciding a fix does not apply to that line, e.g. because the feature it repairs never shipped there. The recorded reason is printed so the next release does not re-open the same investigation. A `dropped` entry reading `no reason recorded` is the one that needs a human. - -## Backports the labels do not account for - -The verdicts start from labels, so on their own they cannot see a backport of a PR nobody labelled. Two further sections start from the other side: every PR on the release branch, and the originals it names by the bot's head branch or in a `Backport of` phrase (see [How landing is established](#how-landing-is-established)). The header carries their counts after a `|`, e.g. `backported=42 | DUPLICATE=3 unlabelled=13`. - -`UNLABELLED` lists each original that backport PRs on the branch claim although it is not a candidate for the branch, with every backport PR claiming it and that PR's state. Each entry also says what those PRs amount to: `backport PR merged` once one of them merged, `claimed here, not merged` while the only live one is still open, and `claimed here, closed unmerged` when every one was closed. A merged claim is only a merged PR, since an unlabelled original is not checked commit by commit; it reads `backport PR merged, confirmed by @login` when the PR carries the [attestation marker](#attesting-a-backport-complete). Most are hand backports of PRs that never carried a label, merged on purpose. An original that does carry a backport request, one that resolved to a different line at merge time, is listed with its labels. The section is informational and never moves the exit code: the gate answers whether everything labelled landed, and an unlabelled backport can only add to a branch, never leave a labelled change off it. - -`DUPLICATE` lists each original claimed by more than one live backport PR, labelled or not: - -| Kind | Meaning | Action | -|------|---------|--------| -| `open-twice` | Two or more backport PRs are open for it — typically the bot's conflict draft next to a hand backport, or a fork PR next to its reopening from a branch in this repository | merge one, close the rest | -| `open-after-merge` | A backport already merged and another one is still open | close the leftover, or merge it if it is the second half of a split backport | - -Both kinds make the audit exit `1`. Neither reaches a verdict, which judges the commits on the branch and names at most one open backport PR, and in both a human has to decide what happens to the open PR before the cut. A closed PR next to an open one is the normal shape of a redone backport and is not flagged, and neither are two merged ones, which are history the branch already carries. - -```console - DUPLICATE -- one original, more than one live backport PR (3): - https://github.com/cozystack/cozystack/pull/3936 - #3936 fix(rabbitmq): right-size the default resources preset to s1.nano - label=kind/backport -- a backport PR still open after another one merged - backport #4372 OPEN (draft) https://github.com/cozystack/cozystack/pull/4372 - backport #4393 MERGED https://github.com/cozystack/cozystack/pull/4393 - - UNLABELLED -- backport PRs here for originals not labelled for this line (13, informational): - https://github.com/cozystack/cozystack/pull/4280 - #4280 fix(kafka): keep the pre-delete hook's credentials as long as its Job - backport PR merged - backport #4421 CLOSED https://github.com/cozystack/cozystack/pull/4421 - backport #4456 MERGED https://github.com/cozystack/cozystack/pull/4456 -``` - -The titles of unlabelled originals are not in any listing the audit already makes, so they come from one GraphQL request covering the whole run. If it fails, the report goes out without them and says so on stderr; the URLs are derived locally and the verdicts and exit code do not depend on it. - -## How a PR is matched to a release line - -A `kind/backport` label does not name a branch. The audit selects merged PRs that currently carry a backport label, then assigns each to the release line selected at its merge time: `kind/backport` means the current line, and `kind/backport-previous` means the previous line. A label added after merge still makes the PR a candidate. [`backport.yaml`](../../.github/workflows/backport.yaml) also runs when a label is added and selects its target at that time, so a late label can make the audit and the bot disagree; see [Limits](#limits). - -What "the current line" means changed once, so the audit carries both rules and picks by the PR's merge date: - -| PR merged | rule `backport.yaml` was running | a line exists from | -|-----------|----------------------------------|--------------------| -| before 2026-08-03 | `getLatestRelease` | its first non-prerelease release publishing | -| on or after 2026-08-03 | the two newest existing `release-X.Y` branches | its `release-X.Y` branch being pushed | - -The cutover is a single moment because both halves landed in one push: 53a7fc4dc made cutting an rc freeze the line into `release-X.Y`, and 6ff5b98d5 pointed the bot at the branch list. Before that a release branch was created at promote time, so the newest branch and the newest published stable named the same line and the two rules could not disagree. After it they disagree for the length of every freeze window — `release-X.Y` is created when the first `vX.Y.0-rc.N` is cut, potentially weeks before `vX.Y.0` publishes, and everything labelled in between belongs to the line being stabilised. That window is exactly when a backport has to reach the branch, because the frozen branch is the only way into the release. - -The audit estimates branch opening from the earliest `vX.Y.0-rc.N` tagged commit dated on or after the cutover. It ignores earlier tags individually; a line with no remaining tag falls back to its first stable release date. [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) creates the branch at the tagged commit, but the commit timestamp is not the branch-push timestamp. PRs merged in that interval can be assigned to the new line before the bot could target it. The rc release's `publishedAt` is later still, after the build finishes. - -Lines are then ranked by version and never by when they opened, matching the numeric descending sort the workflow applies to its branch list. The two orders come apart as soon as a freeze overlaps the previous line's stabilisation, and version order is also what keeps `release-1.10` above `release-1.9`. - -The labels themselves were namespaced under `kind/` in 7b71053a0, which renamed them in place via label-sync aliases, so every historical PR reads back under the new name. The bare `backport` and `backport-previous` are still queried alongside them, because the org-level dosubot goes on applying the old spelling to new PRs — the same transitional allowance [`backport.yaml`](../../.github/workflows/backport.yaml) makes, and it retires at the same time. A PR found under either spelling is reported under the `kind/` one. - -## How landing is established - -Evidence is weighed per commit, because a commit is what a backport can lose: - -1. **Reachability.** The PR's merge commit is reachable from the release branch, i.e. it merged before the branch was cut (or `main` was later merged in). Nothing was ever needed: `in-branch`. -2. **A linked backport PR.** Found by the bot's head branch `backport--to-release-X.Y`, or by a `Backport of #N` reference in the body, which is what a hand-written backport carries. It never outranks the commits (item 3): a merged backport PR with one of the PR's commits missing is `partial`, because the bot's conflict drafts stop at the first commit that does not apply and drop the rest, so a merged draft looks finished at PR level. `MERGED` settles it alone only for a PR of a single commit — including on a `--no-fetch` run whose local refs predate the backport's merge. For a PR of several commits, none of them found and a merged backport PR is `unverified`. `OPEN` is `pending`; `CLOSED` is `dropped`. - - A backport carrying several changes links to every original its `Backport of` phrase names: a list (`Backport of #3938 and #4280`, `Backport of #1, #2, and #3`), and the `Backport of #4253 to release-1.6, together with #3460` form used for a dependency pulled along. Only that phrase is read, so the issue a backport fixes or a CI run it cites further on is never taken for an original. A list counts in full only when it visibly ends — at the end of the line or the sentence, or where the `to release-X.Y` clause starts. One that runs on into anything else (`Backport of #10, #20 is not included`) may be saying something about its later items, so only its first reference counts. A reference qualified with this repository (`cozystack/cozystack#N`, any case) counts like a bare `#N`, and one qualified with any other repository is skipped, because reading `other/repo#20` as local #20 would let an unrelated backport vouch for whichever local PR carries that number. -3. **The branch's own history, commit by commit.** Every commit the PR contributed has to be on the branch; merge commits and commits that change nothing are skipped, since the backport bot drops both. A commit counts when it is reachable, when an `-x` cherry-pick reference names it, or when a commit with its subject is there. Subject and `-x` evidence comes only from commits on the release branch that are not reachable from `/main`; shared history still counts for direct reachability. Subjects are matched one to one: two commits with the same subject, `fix tests` say, need two branch commits of that name, and a branch commit whose `-x` reference names any other commit than the PR's merge commit is evidence only through that reference. Nothing weaker is taken: not an `-x` reference to the PR's merge commit, which survives a cherry-pick later amended to drop a commit, and not matching lines, which an unrelated line of the same text satisfies. A wrong `backported` is the one answer a release gate must not give, while a false alarm costs a look and can be settled by an attestation. All commits is `backported`, some is `partial`. This is also what catches a hand backport nobody linked. - -### Attesting a backport complete - -Some complete backports cannot be shown commit by commit, most often a fork contributor's backport squashed into one commit that carries only the first commit's subject. A maintainer who has checked such a backport can say so on the merged backport PR, with a comment that consists of this single line and nothing else: - -```text -backport-audit: complete -``` - -That turns the candidate's `partial` or `unverified` into `confirmed`, which does not fail the gate; the report names the author, links the comment, and keeps listing the commits nothing on the branch names. - -The marker is an explicit attestation by whoever wrote it, never something the audit infers. It counts only when it is the whole comment: exactly `backport-audit: complete`, in that case, starting at the beginning of its line, with nothing but blank lines before it and nothing but whitespace after it. Any other text makes the comment something other than an attestation, so a sentence such as "do not post backport-audit: complete yet", a quote or a code block never counts, and neither does the marker with an explanation next to it; post the explanation as a separate comment. It counts only on a backport PR that merged, since an open or closed one put nothing on the branch to vouch for. And it counts only from someone with a hand in the repository: the comment's GitHub author association has to be `OWNER`, `MEMBER` or `COLLABORATOR`, and comments from automation are ignored whatever their association — logins ending in `[bot]`, and the review and dependency bots gh reports without that suffix, such as `coderabbitai`, `dependabot`, `renovate` and `copilot`. A review bot quoting the marker, or a contributor vouching for their own backport, confirms nothing. - -## Machine-readable output - -```bash -# URLs of what never landed, ready to paste into a tracking issue -go run ./cmd/backport-audit --json release-1.5 | jq -r '.[].candidates[] | select(.status=="MISSING") | .url' - -# every open backport PR the audit links, labelled or not, to merge or close, across lines -go run ./cmd/backport-audit --json release-1.6 release-1.5 \ - | jq -r '[.[] | (.candidates[], .unlabelled[]) | .backport_prs[] | select(.state=="OPEN") | .url] | unique[]' - -# dropped items with the reason someone recorded -go run ./cmd/backport-audit --json release-1.4 \ - | jq -r '.[].candidates[] | select(.status=="dropped") | "#\(.number)\t\(.reason)"' - -# commits a partial backport still lacks, one per line -go run ./cmd/backport-audit --json release-1.6 \ - | jq -r '.[].candidates[] | select(.status=="partial") | "#\(.number)\t\(.missing_commits[] | "\(.oid[0:9]) \(.subject)")"' - -# open backport PRs competing for one original, to settle before the cut -go run ./cmd/backport-audit --json release-1.6 \ - | jq -r '.[].duplicates[] | "#\(.number) \(.kind): \([.backport_prs[] | select(.state=="OPEN") | .url] | join(" "))"' -``` - -Output is an object keyed by branch, each holding three arrays: `candidates`, `unlabelled` and `duplicates`, each always present and possibly empty. `partial`, `unverified` and `confirmed` candidates carry `missing_commits`, each an `oid` and a `subject`, and a `confirmed` one also carries `confirmation`: `by`, `url` and `backport_pr`. An `unlabelled` record carries the same `confirmation` when its merged backport PR has the marker. A `candidates` record: - -```json -{ - "number": 3963, - "title": "fix(clickhouse): scheme the backup S3_ENDPOINT on the system-bucket flow", - "url": "https://github.com/cozystack/cozystack/pull/3963", - "author": "androndo", - "mergedAt": "2026-08-28T11:05:15Z", - "label": "kind/backport", - "backport_prs": [{"number": 3987, "state": "OPEN", "url": "..."}], - "status": "pending", - "evidence": "backport PR #3987 open" -} -``` - -An `unlabelled` record, where `labels` lists the backport requests the original carries for other lines and is empty when it carries none: - -```json -{ - "number": 4280, - "title": "fix(kafka): keep the pre-delete hook's credentials as long as its Job", - "url": "https://github.com/cozystack/cozystack/pull/4280", - "labels": [], - "backport_prs": [ - {"number": 4421, "state": "CLOSED", "url": "..."}, - {"number": 4456, "state": "MERGED", "url": "..."} - ] -} -``` - -A `duplicates` record, where `label` is the request the original was audited under on this line and is empty when it is not a candidate here: - -```json -{ - "number": 4254, - "title": "feat(kubevirt): expose migration configuration through platform values", - "url": "https://github.com/cozystack/cozystack/pull/4254", - "label": "kind/backport", - "kind": "open-twice", - "backport_prs": [ - {"number": 4322, "state": "OPEN", "url": "...", "draft": true}, - {"number": 4339, "state": "OPEN", "url": "..."}, - {"number": 4402, "state": "OPEN", "url": "..."} - ] -} -``` - -## Limits - -A `MISSING`, `partial` or `unverified` verdict is a prompt to check, not proof of absence. A hand backport squashed or reworded without `-x` reads as `partial` or `unverified` until someone attests it complete, and as `MISSING` when nothing links it to its original; confirm at diff level before redoing the work. And the audit reports *that* something is missing, never *why* — a run of `MISSING` entries clustered in time usually means the bot itself was failing during that window, which is worth checking before cherry-picking them one by one. - -The audit reads labels as they stand when it runs. A label added afterwards is seen only by the next run, and until then its PR is either absent or, if someone already backported it, listed as `UNLABELLED` rather than audited, so run it right before the cut rather than once at the start of the day. - -A label is always read against the release lines as they stood when the PR merged, however much later the label was added, because nothing the audit lists records when that happened. A PR merged on 2026-07-01 and labelled `kind/backport` a month later, after `v1.6.0` published, is audited as a `release-1.5` candidate. [`backport.yaml`](../../.github/workflows/backport.yaml) runs again when the label is added and resolves its targets at that moment, so it opens the backport against `release-1.6`. When the line has moved on between the merge and the label, the two therefore disagree: the audit expects the change on the merge-time line and reports it `MISSING` there unless it arrived some other way, and lists the bot's backport under `UNLABELLED` on the newer one. `TestCandidateLabel` pins the audit's side of this. - -A backport PR that names its original neither by the bot's head branch nor in a `Backport of` phrase links to nothing, so it is invisible to both cross-checks: it is not listed as `UNLABELLED` and does not count towards a `DUPLICATE`. For a labelled original the branch history can still prove the change landed, through an identical commit subject or an `-x` cherry-pick reference. For an unlabelled one those are not read at all, so a backport of an unlabelled PR that only a cherry-pick trailer points to, or nothing does, appears nowhere in the report. diff --git a/cmd/backport-audit/main.go b/cmd/backport-audit/main.go deleted file mode 100644 index cee7aa791..000000000 --- a/cmd/backport-audit/main.go +++ /dev/null @@ -1,1569 +0,0 @@ -/* -Copyright 2026 The Cozystack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Command backport-audit reports outstanding labelled changes before a release -// cut. It reads GitHub and local Git history, fetching refs unless --no-fetch is -// set. The README describes candidate selection, delivery evidence and limits. -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "regexp" - "slices" - "sort" - "strconv" - "strings" - "time" -) - -const usageText = `Audit whether every backport-labelled PR actually landed on a release branch. - -Exits 0 when nothing is outstanding and 1 when something is, so it can gate a release. -Backports of PRs that carry no label for the line are listed without affecting the -exit code; an original with more than one live backport PR counts as outstanding. - -Usage: - backport-audit [OPTIONS] [release-X.Y ...] - -Arguments: - release-X.Y ... Release lines to audit (default: the two newest) - -Options: - --remote NAME Git remote holding the release branches (default: origin) - --limit N Max merged PRs to scan per label (default and ceiling: 1000) - --no-fetch Trust the local refs as-is, skipping git fetch - --json Machine-readable output - --no-color Disable color output (auto-disabled on non-TTY) - -h, --help Show this help -` - -// A reference to a PR, optionally qualified by its repository, and a list of -// them as prose writes one: "#1 and #2", "#1, #2 and #3", "#1, #2, and #3". -const ( - prRef = `(?:[\w.-]+/[\w.-]+)?#\d+` - prRefList = prRef + `(?:(?:\s*,\s*(?:and\s+)?|\s+and\s+)` + prRef + `)*` -) - -// A release line named in prose, bare or in backticks, as a whole word: -// release-1.6-fixes, release-1.6.1 and release-1.6.fixes start with a line name -// without being one, so punctuation ends the name only when a space or the end -// of the text follows it. -const releaseLineName = `(?:\x60release-\d+\.\d+\x60|release-\d+\.\d+)(?:$|\s|\)|[.,;:](?:\s|$))` - -// upstreamRepo is the repository an unqualified reference is read against when -// a PR's URL does not say which repository it belongs to. -const upstreamRepo = "cozystack/cozystack" - -var ( - releaseLineRE = regexp.MustCompile(`^release-(\d+)\.(\d+)$`) - backportHeadRE = regexp.MustCompile(`^backport-(\d+)-to-(release-\d+\.\d+)$`) - // A hand backport that carries more than one change names all of them in - // its "Backport of" phrase, either as a list or as the "to release-X.Y, - // together with #N" this repository writes for a dependency pulled along. - backportOfRE = regexp.MustCompile(`[Bb]ackport of\s+(` + prRefList + `)` + - `(?:\s+to\s+\x60?release-\d+\.\d+\x60?,?\s+together with\s+(` + prRefList + `))?`) - // What a list of references may be followed by for every item in it to be - // an original: the end of the line, the end of a sentence, or the "to - // release-X.Y" that names the target. Anything else may be the list going - // on to say something about its later items -- "#10, #20 is not included", - // "#10, #20 to follow in a separate PR" -- so an unterminated list keeps - // only its first item, the one the phrase names directly. - prRefListEndRE = regexp.MustCompile(`^(?:[ \t]*(?:\r?\n|$)|\.(?:\s|$)|\s+to\s+` + releaseLineName + `)`) - prRefRE = regexp.MustCompile(`(?:([\w.-]+/[\w.-]+))?#(\d+)`) - cherryPickRE = regexp.MustCompile(`cherry picked from commit ([0-9a-f]{7,40})`) - releaseTagRE = regexp.MustCompile(`^v(\d+)\.(\d+)\.(\d+)$`) - // The tag whose cut creates release-X.Y. cut-prerelease.yaml gates the - // freeze on kind == rc and patch == 0, so alpha, beta and patch-line rcs - // deliberately do not match. - freezeTagRE = regexp.MustCompile(`^v(\d+)\.(\d+)\.0-rc\.\d+$`) -) - -// Caps on the three listings the audit makes. gh truncates at --limit in -// silence, so each one is checked against its cap by truncated() and the -// numbers are set well clear of the current populations rather than close to -// them: at the time of writing, 290 labelled PRs, 217 releases, and at most 54 -// PRs on any one release branch. Raising a cap costs nothing when it is not -// reached -- gh stops as soon as the result set is exhausted -- so the reason -// they are bounded at all is to keep a runaway query from paginating forever. -const ( - releaseListCap = 1000 - branchPRCap = 1000 -) - -// searchResultCap is the most results GitHub search returns for one query, -// however far it is paginated. gh pr list answers a --label filter through -// search, so the candidate listing stops here whatever --limit asks for, and -// gh's --json output does not warn that it did. The other two listings do not -// go through search and are bounded by their own caps alone. -const searchResultCap = 1000 - -// candidateCeiling is the listing length at which a label query with --limit -// limit has to be taken as truncated, and what to do about it. -func candidateCeiling(limit int) (int, string) { - if limit < searchResultCap { - return limit, "re-run with a higher --limit" - } - return searchResultCap, "GitHub search returns at most 1000 results per query; split the label query in cmd/backport-audit, e.g. by merge date" -} - -// freezeContractLandedAt is when cutting an rc started creating release-X.Y, -// and with it when backport.yaml stopped resolving its targets from the last -// published stable. -// -// One moment, not two: 53a7fc4dc ("cutting an rc freezes the line into -// release-X.Y") and 6ff5b98d5 ("target the newest existing release line") -// landed on main in the same push, which is what makes a single cutover -// faithful. Before it a release-X.Y branch was created at promote time, so the -// newest branch and the newest published stable named the same line and the -// two rules could not disagree. After it they disagree for the length of every -// freeze window, which is precisely when a backport has to reach the branch. -var freezeContractLandedAt = time.Date(2026, 8, 3, 16, 31, 59, 0, time.UTC) - -// The two backport requests .github/workflows/backport.yaml acts on, named by -// the label that carries them. These names are what the audit reports. -const ( - labelCurrent = "kind/backport" - labelPrevious = "kind/backport-previous" -) - -// backportLabels maps each request to every label spelling that expresses it. -// -// TRANSITIONAL, matching the same allowance in backport.yaml: the labels were -// namespaced under kind/ in 7b71053a0, carrying the bare names as label-sync -// aliases so the rename applied in place. Historical PRs therefore read back as -// `kind/backport` today, but the org-level dosubot still applies the bare names -// on new PRs, so a candidate can carry either. Both are queried and folded onto -// the canonical name above. When dosubot's PR labelling is switched off, the -// legacy spellings here and the ones in backport.yaml can be dropped together. -var backportLabels = []struct { - canonical string - spellings []string -}{ - {labelCurrent, []string{labelCurrent, "backport"}}, - {labelPrevious, []string{labelPrevious, "backport-previous"}}, -} - -// Statuses, most to least in need of attention. outstanding is the prefix of -// this order that makes the audit exit non-zero. -const ( - statusMissing = "MISSING" - statusPartial = "partial" - statusUnverified = "unverified" - statusPending = "pending" - statusDropped = "dropped" - statusConfirmed = "confirmed" - statusBackported = "backported" - statusInBranch = "in-branch" -) - -var ( - statusOrder = []string{statusMissing, statusPartial, statusUnverified, statusPending, statusDropped, - statusConfirmed, statusBackported, statusInBranch} - outstanding = []string{statusMissing, statusPartial, statusUnverified, statusPending, statusDropped} - headings = map[string]string{ - statusMissing: "MISSING -- labelled for this line, no trace of it here", - statusPartial: "PARTIAL -- some of the PR's commits are here, not all of them", - statusUnverified: "UNVERIFIED -- backport PR merged, none of the PR's commits found here", - statusPending: "PENDING -- backport PR open, not merged", - statusDropped: "DROPPED -- backport PR closed without merging", - } -) - -// The two ways one original can be claimed by more than one live backport PR on -// a branch. Both are outstanding. -const ( - dupOpenTwice = "open-twice" - dupOpenAfterMerge = "open-after-merge" -) - -var dupDescriptions = map[string]string{ - dupOpenTwice: "more than one backport PR open", - dupOpenAfterMerge: "a backport PR still open after another one merged", -} - -type config struct { - remote string - limit int - fetch bool - asJSON bool - useColor bool - branches []string - - red, green, yellow, cyan, dim, bold, reset string -} - -// line is a release line, i.e. a major.minor pair with a release-X.Y branch. -type line struct{ major, minor int } - -func (l line) branch() string { return fmt.Sprintf("release-%d.%d", l.major, l.minor) } -func (l line) less(o line) bool { - if l.major != o.major { - return l.major < o.major - } - return l.minor < o.minor -} - -// mainPR is a merged-to-main PR carrying at least one backport label. -type mainPR struct { - Number int `json:"number"` - Title string `json:"title"` - URL string `json:"url"` - MergedAt time.Time `json:"mergedAt"` - MergeCommit struct { - Oid string `json:"oid"` - } `json:"mergeCommit"` - Author struct { - Login string `json:"login"` - } `json:"author"` - Labels []struct { - Name string `json:"name"` - } `json:"labels"` - - labels map[string]bool -} - -// backportPR is a PR opened against a release branch. -type backportPR struct { - Number int `json:"number"` - Title string `json:"title"` - URL string `json:"url"` - State string `json:"state"` - HeadRefName string `json:"headRefName"` - Body string `json:"body"` - IsDraft bool `json:"isDraft"` - Comments []prComment `json:"comments"` -} - -// prComment is one comment on a backport PR. -type prComment struct { - Author struct { - Login string `json:"login"` - } `json:"author"` - AuthorAssociation string `json:"authorAssociation"` - Body string `json:"body"` - URL string `json:"url"` -} - -// linkedPR is the trimmed view of a backport PR carried in the output. -type linkedPR struct { - Number int `json:"number"` - State string `json:"state"` - URL string `json:"url"` - Draft bool `json:"draft,omitempty"` -} - -func trimPR(b backportPR) linkedPR { - return linkedPR{Number: b.Number, State: b.State, URL: b.URL, Draft: b.IsDraft} -} - -// verdict is one audited candidate. The JSON field names are the tool's -// contract with jq one-liners, so they are spelled explicitly. -type verdict struct { - Number int `json:"number"` - Title string `json:"title"` - URL string `json:"url"` - Author string `json:"author"` - MergedAt string `json:"mergedAt"` - Label string `json:"label"` - BackportPRs []linkedPR `json:"backport_prs"` - Status string `json:"status"` - Evidence string `json:"evidence"` - Reason string `json:"reason,omitempty"` - // MissingCommits are the PR's commits a partial or unverified backport - // shows no trace of. - MissingCommits []prCommit `json:"missing_commits,omitempty"` - Confirmation *confirmation `json:"confirmation,omitempty"` -} - -// prCommit is one commit a PR contributed to main. -type prCommit struct { - Oid string `json:"oid"` - Subject string `json:"subject"` -} - -// unlabelledBackport is an original that backport PRs on the branch claim -// although it is not a candidate for the branch. Labels holds the backport -// requests it does carry, which then resolved to some other line. -type unlabelledBackport struct { - Number int `json:"number"` - Title string `json:"title"` - URL string `json:"url"` - Labels []string `json:"labels"` - BackportPRs []linkedPR `json:"backport_prs"` - Confirmation *confirmation `json:"confirmation,omitempty"` -} - -// duplicateBackport is an original claimed by more than one live backport PR -// on the branch. Label is the request it was audited under here, empty when it -// is not a candidate for the branch. -type duplicateBackport struct { - Number int `json:"number"` - Title string `json:"title"` - URL string `json:"url"` - Label string `json:"label"` - Kind string `json:"kind"` - BackportPRs []linkedPR `json:"backport_prs"` -} - -// branchReport is everything the audit found on one release branch. -type branchReport struct { - Candidates []verdict `json:"candidates"` - Unlabelled []unlabelledBackport `json:"unlabelled"` - Duplicates []duplicateBackport `json:"duplicates"` -} - -// clean reports whether the branch passes the gate. Unlabelled backports do not -// enter into it: they report claims outside the labelled candidate set. -func (r *branchReport) clean() bool { - for _, v := range r.Candidates { - if slices.Contains(outstanding, v.Status) { - return false - } - } - return len(r.Duplicates) == 0 -} - -func main() { os.Exit(run()) } - -func run() int { - cfg, code, done := parseArgs(os.Args[1:]) - if done { - return code - } - - if cfg.fetch { - fmt.Fprintln(os.Stderr, "fetching refs...") - if _, err := capture("git", "fetch", "--quiet", "--tags", cfg.remote); err != nil { - fmt.Fprintf(os.Stderr, "git fetch failed: %v\n", err) - return 2 - } - } - - if len(cfg.branches) == 0 { - lines, err := releaseLines(cfg.remote) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return 2 - } - if len(lines) == 0 { - fmt.Fprintf(os.Stderr, "no release-X.Y branches found on %s\n", cfg.remote) - return 2 - } - // Newest first, at most two. - for i := len(lines) - 1; i >= 0 && len(cfg.branches) < 2; i-- { - cfg.branches = append(cfg.branches, lines[i].branch()) - } - fmt.Fprintf(os.Stderr, "auditing %s\n", strings.Join(cfg.branches, ", ")) - } - - for _, b := range cfg.branches { - if !releaseLineRE.MatchString(b) { - fmt.Fprintf(os.Stderr, "not a release line: %s (expected release-.)\n", b) - return 2 - } - if !gitOK("rev-parse", "--verify", "--quiet", cfg.remote+"/"+b) { - fmt.Fprintf(os.Stderr, "no such branch: %s/%s\n", cfg.remote, b) - return 2 - } - } - - opened, err := lineOpenDates() - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return 2 - } - cands, err := candidates(cfg.limit) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return 2 - } - fmt.Fprintf(os.Stderr, "%d merged PRs carry a backport label\n", len(cands)) - - out := map[string]*branchReport{} - clean := true - for _, branch := range cfg.branches { - rep, err := cfg.audit(branch, cands, opened) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return 2 - } - out[branch] = rep - // The exit code is the whole point of running this in a gate, so it is - // computed from the results, not as a side effect of printing them. - if !rep.clean() { - clean = false - } - } - - // A title is decoration, so a failed lookup costs the titles and nothing - // else: the verdicts and the exit code are already settled above. - if err := fillTitles(out); err != nil { - fmt.Fprintf(os.Stderr, "could not look up titles of unlabelled originals: %v\n", err) - } - - if cfg.asJSON { - blob, err := json.MarshalIndent(out, "", " ") - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return 2 - } - fmt.Println(string(blob)) - } else { - for _, branch := range cfg.branches { - cfg.report(branch, out[branch]) - } - fmt.Println() - } - - if clean { - return 0 - } - return 1 -} - -func parseArgs(args []string) (*config, int, bool) { - cfg := &config{remote: "origin", limit: 1000, fetch: true, useColor: true} - for i := 0; i < len(args); i++ { - a := args[i] - next := func() (string, bool) { - if i+1 >= len(args) { - fmt.Fprintf(os.Stderr, "%s requires a value\n", a) - return "", false - } - i++ - return args[i], true - } - switch a { - case "-h", "--help": - fmt.Print(usageText) - return nil, 0, true - case "--remote": - v, ok := next() - if !ok { - return nil, 2, true - } - cfg.remote = v - case "--limit": - v, ok := next() - if !ok { - return nil, 2, true - } - n, err := strconv.Atoi(v) - if err != nil || n <= 0 { - fmt.Fprintf(os.Stderr, "invalid --limit value: %s\n", v) - return nil, 2, true - } - cfg.limit = n - case "--no-fetch": - cfg.fetch = false - case "--json": - cfg.asJSON = true - case "--no-color": - cfg.useColor = false - default: - if strings.HasPrefix(a, "-") { - fmt.Fprintf(os.Stderr, "Unknown argument: %s\n", a) - fmt.Fprint(os.Stderr, usageText) - return nil, 2, true - } - cfg.branches = append(cfg.branches, a) - } - } - - // Color is for a human reading a terminal; --json and pipes get none. - if cfg.asJSON { - cfg.useColor = false - } - if fi, err := os.Stdout.Stat(); err != nil || fi.Mode()&os.ModeCharDevice == 0 { - cfg.useColor = false - } - cfg.setColors() - return cfg, 0, false -} - -func (cfg *config) setColors() { - if cfg.useColor { - cfg.red = "\033[0;31m" - cfg.green = "\033[0;32m" - cfg.yellow = "\033[1;33m" - cfg.cyan = "\033[0;36m" - cfg.dim = "\033[2m" - cfg.bold = "\033[1m" - cfg.reset = "\033[0m" - } -} - -// colorFor is the accent a status is reported in. -func (cfg *config) colorFor(status string) string { - switch status { - case statusMissing, statusPartial, statusUnverified: - return cfg.red - case statusPending: - return cfg.yellow - case statusDropped: - return cfg.cyan - case statusConfirmed, statusBackported, statusInBranch: - return cfg.green - } - return "" -} - -func capture(name string, args ...string) (string, error) { - out, err := exec.Command(name, args...).Output() - if err != nil { - if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { - return "", fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(ee.Stderr))) - } - return "", fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) - } - return string(out), nil -} - -// git runs a git command whose failure means the audit cannot be trusted. -func git(args ...string) (string, error) { return capture("git", args...) } - -func gitOK(args ...string) bool { - return exec.Command("git", args...).Run() == nil -} - -// truncated turns a saturated `gh ... --limit N` query into an error instead of -// a quietly shorter answer. -// -// Every list this tool makes is a completeness claim -- these are all the -// candidates, this is every backport PR on the branch -- and gh stops at -// --limit without saying so. A saturated query therefore does not make the -// audit partial, it makes its verdicts wrong: the PRs past the cut are -// reported nowhere, and the exit code says clean. That is the same failure -// class the tool exists to catch, so it fails loudly instead. -func truncated(got, limit int, what, remedy string) error { - if got < limit { - return nil - } - return fmt.Errorf("%s hit its cap of %d results, so the list is truncated and the audit would be unsound: %s", what, limit, remedy) -} - -func ghJSON(target any, args ...string) error { - out, err := capture("gh", args...) - if err != nil { - return err - } - return json.Unmarshal([]byte(out), target) -} - -// releaseLines returns the two-component release branches on the remote, -// oldest first. release-1.6.0-rc.1 and the like are staging branches, not -// maintenance lines, so they are excluded. -func releaseLines(remote string) ([]line, error) { - out, err := git("branch", "-r", "--list", remote+"/release-*") - if err != nil { - return nil, err - } - seen := map[line]bool{} - for raw := range strings.SplitSeq(out, "\n") { - name := strings.TrimPrefix(strings.TrimSpace(raw), remote+"/") - if m := releaseLineRE.FindStringSubmatch(name); m != nil { - major, _ := strconv.Atoi(m[1]) - minor, _ := strconv.Atoi(m[2]) - seen[line{major, minor}] = true - } - } - lines := make([]line, 0, len(seen)) - for l := range seen { - lines = append(lines, l) - } - sort.Slice(lines, func(i, j int) bool { return lines[i].less(lines[j]) }) - return lines, nil -} - -type lineOpen struct { - line line - when time.Time -} - -// Stable publication dates cover lines opened before the freeze contract; -// tagged-commit dates estimate openings since then. Both eras are needed when -// resolving a historical backport label against the lines available at merge. -func lineOpenDates() ([]lineOpen, error) { - var releases []struct { - TagName string `json:"tagName"` - PublishedAt time.Time `json:"publishedAt"` - IsPrerelease bool `json:"isPrerelease"` - IsDraft bool `json:"isDraft"` - } - if err := ghJSON(&releases, "release", "list", "--limit", strconv.Itoa(releaseListCap), - "--json", "tagName,publishedAt,isPrerelease,isDraft"); err != nil { - return nil, err - } - if err := truncated(len(releases), releaseListCap, "the release listing", - "raise releaseListCap in cmd/backport-audit"); err != nil { - return nil, err - } - - earliest := map[line]time.Time{} - for _, rel := range releases { - if rel.IsPrerelease || rel.IsDraft { - continue - } - m := releaseTagRE.FindStringSubmatch(rel.TagName) - if m == nil { - continue - } - major, _ := strconv.Atoi(m[1]) - minor, _ := strconv.Atoi(m[2]) - l := line{major, minor} - if prev, ok := earliest[l]; !ok || rel.PublishedAt.Before(prev) { - earliest[l] = rel.PublishedAt - } - } - - frozen, err := freezeDates() - if err != nil { - return nil, err - } - for l, when := range frozen { - earliest[l] = when - } - - opened := make([]lineOpen, 0, len(earliest)) - for l, when := range earliest { - opened = append(opened, lineOpen{l, when}) - } - sort.Slice(opened, func(i, j int) bool { return opened[i].when.Before(opened[j].when) }) - return opened, nil -} - -// freezeDates estimates line opening from the earliest vX.Y.0-rc.N tagged -// commit at or after the freeze-contract cutover. Earlier tags are ignored; -// lines with no remaining tag use the published-stable date in lineOpenDates. -// The tagged commit can predate the branch push; see the README's limits. -func freezeDates() (map[line]time.Time, error) { - out, err := git("for-each-ref", "--format=%(refname:strip=2) %(committerdate:iso-strict)", "refs/tags/v*") - if err != nil { - return nil, err - } - return parseFreezeDates(out) -} - -// parseFreezeDates is the half of freezeDates that does not need a repository: -// ` ` per line, in, freeze moments out. -func parseFreezeDates(out string) (map[line]time.Time, error) { - frozen := map[line]time.Time{} - for raw := range strings.SplitSeq(out, "\n") { - name, date, ok := strings.Cut(strings.TrimSpace(raw), " ") - if !ok { - continue - } - m := freezeTagRE.FindStringSubmatch(name) - if m == nil { - continue - } - when, err := time.Parse(time.RFC3339, date) - if err != nil { - return nil, fmt.Errorf("tag %s: unparseable date %q: %w", name, date, err) - } - if when.Before(freezeContractLandedAt) { - continue - } - major, _ := strconv.Atoi(m[1]) - minor, _ := strconv.Atoi(m[2]) - l := line{major, minor} - if prev, ok := frozen[l]; !ok || when.Before(prev) { - frozen[l] = when - } - } - return frozen, nil -} - -// targetsAt reports the current and previous release lines as of when, which is -// what the `kind/backport` and `kind/backport-previous` labels meant at that -// moment. -// -// The lines that exist at that point are ranked by version, not by when they -// opened, matching the numeric descending sort backport.yaml applies to its -// branch list. The two orders normally coincide, and stop coinciding as soon -// as a freeze overlaps the previous line's stabilisation: cut vX.(Y+1).0-rc.1 -// while vX.Y.0 is still unpublished and the newer line opens first in time -// while still being the newer line. Ranking by version is also what keeps -// release-1.10 above release-1.9. -func targetsAt(opened []lineOpen, when time.Time) (current, previous *line) { - var live []line - for _, o := range opened { - if !o.when.After(when) { - live = append(live, o.line) - } - } - sort.Slice(live, func(i, j int) bool { return live[j].less(live[i]) }) - if n := len(live); n > 0 { - current = &live[0] - if n > 1 { - previous = &live[1] - } - } - return current, previous -} - -// candidates returns the merged-to-main PRs carrying a backport label, keyed by -// PR number so a PR matched under two spellings of the same request, or under -// both requests, is one entry carrying both canonical names. -func candidates(limit int) (map[int]*mainPR, error) { - found := map[int]*mainPR{} - for _, req := range backportLabels { - for _, spelling := range req.spellings { - var prs []mainPR - if err := ghJSON(&prs, "pr", "list", "--base", "main", "--state", "merged", - "--label", spelling, "--limit", strconv.Itoa(limit), - "--json", "number,title,url,mergedAt,mergeCommit,labels,author"); err != nil { - return nil, err - } - ceiling, remedy := candidateCeiling(limit) - if err := truncated(len(prs), ceiling, "the merged-PR query for label "+spelling, remedy); err != nil { - return nil, err - } - for i := range prs { - pr := prs[i] - existing, ok := found[pr.Number] - if !ok { - pr.labels = map[string]bool{} - found[pr.Number] = &pr - existing = &pr - } - existing.labels[req.canonical] = true - } - } - } - return found, nil -} - -type branchHistory struct { - branch string - ref string - commits map[string]bool - // Only commits outside main supply subject and -x evidence. Shared - // ancestors can have the same subject without carrying the backport. - subjects map[string][]string - refs map[string][]string -} - -func newBranchHistory(remote, branch string) (*branchHistory, error) { - h := &branchHistory{ - branch: branch, - ref: remote + "/" + branch, - commits: map[string]bool{}, - subjects: map[string][]string{}, - refs: map[string][]string{}, - } - out, err := git("rev-list", h.ref) - if err != nil { - return nil, err - } - for oid := range strings.FieldsSeq(out) { - h.commits[oid] = true - } - out, err = git("log", "--format=%x00%H%x01%s%x01%b", h.ref, "--not", remote+"/main") - if err != nil { - return nil, err - } - for entry := range strings.SplitSeq(out, "\x00") { - if strings.TrimSpace(entry) == "" { - continue - } - parts := strings.SplitN(entry, "\x01", 3) - for len(parts) < 3 { - parts = append(parts, "") - } - oid, subject := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) - h.subjects[subject] = append(h.subjects[subject], oid) - for _, m := range cherryPickRE.FindAllStringSubmatch(parts[2], -1) { - h.refs[oid] = append(h.refs[oid], m[1]) - } - } - return h, nil -} - -func (h *branchHistory) hasSubject(subject string) bool { - return len(h.subjects[strings.TrimSpace(subject)]) > 0 -} - -func (h *branchHistory) contains(commit string) bool { - return commit != "" && h.commits[commit] -} - -// sameCommit reports whether two object names can be the same commit: an -x -// reference may abbreviate, so either may be a prefix of the other. -func sameCommit(a, b string) bool { - return a != "" && b != "" && (strings.HasPrefix(a, b) || strings.HasPrefix(b, a)) -} - -// missingFrom returns the commits of the PR merged as mergeCommit that the -// branch does not carry. A commit is carried when it is on the branch itself, -// when a branch commit's -x reference names it, or when a branch commit with -// its subject is left over for it. -// -// Subjects are spent one to one: two commits both called "fix tests" need two -// branch commits of that name, not the one they share. One of the PR's own -// commits on the branch is spent on itself, and a branch commit whose -x -// reference names anything but the PR's merge commit -- one of the PR's -// commits included -- is evidence only through that reference: it has said -// what it is a copy of. -func (h *branchHistory) missingFrom(commits []prCommit, mergeCommit string) []prCommit { - spent := map[string]bool{} - carried := make([]bool, len(commits)) - for i, c := range commits { - if h.contains(c.Oid) { - carried[i], spent[c.Oid] = true, true - } - for _, refs := range h.refs { - for _, r := range refs { - if sameCommit(c.Oid, r) { - carried[i] = true - } - } - } - } - var missing []prCommit - for i, c := range commits { - for _, b := range h.subjects[strings.TrimSpace(c.Subject)] { - if carried[i] { - break - } - if !spent[b] && h.namesOnly(b, mergeCommit) { - carried[i], spent[b] = true, true - } - } - if !carried[i] { - missing = append(missing, c) - } - } - return missing -} - -// namesOnly reports whether every -x reference branch commit b carries, if it -// carries any, names commit. -func (h *branchHistory) namesOnly(b, commit string) bool { - for _, r := range h.refs[b] { - if !sameCommit(r, commit) { - return false - } - } - return true -} - -// backportPRsFor indexes every PR on branch by the main PRs it claims to -// backport; see claimedOrigins. -func backportPRsFor(branch string) (map[int][]backportPR, error) { - var prs []backportPR - if err := ghJSON(&prs, "pr", "list", "--base", branch, "--state", "all", "--limit", strconv.Itoa(branchPRCap), - "--json", "number,title,url,state,headRefName,body,isDraft,comments"); err != nil { - return nil, err - } - if err := truncated(len(prs), branchPRCap, "the PR listing for "+branch, - "raise branchPRCap in cmd/backport-audit"); err != nil { - return nil, err - } - return indexByOrigin(prs, branch), nil -} - -func indexByOrigin(prs []backportPR, branch string) map[int][]backportPR { - index := map[int][]backportPR{} - for _, pr := range prs { - for _, n := range claimedOrigins(pr, branch) { - index[n] = append(index[n], pr) - } - } - return index -} - -// claimedOrigins reads the main PRs a PR on branch says it backports: the one -// in the bot's head branch name, and every one in the body's "Backport of" -// phrases, which is what a hand-written backport carries. Hand backports often -// reuse the bot's head naming, so the two usually agree and are counted once. -func claimedOrigins(pr backportPR, branch string) []int { - var origins []int - if m := backportHeadRE.FindStringSubmatch(pr.HeadRefName); m != nil && m[2] == branch { - n, _ := strconv.Atoi(m[1]) - origins = append(origins, n) - } - for _, n := range bodyOrigins(pr.Body, repoOf(pr.URL)) { - if !slices.Contains(origins, n) { - origins = append(origins, n) - } - } - return origins -} - -// bodyOrigins returns the PR numbers named in a body's "Backport of" phrases, in -// order of appearance and without repeats. -// -// A reference qualified with any repository but repo is skipped rather than -// read as a local number: "other/repo#20" linked as #20 would let an unrelated -// backport vouch for whatever local PR happens to carry that number, and a -// merged one would turn its MISSING verdict into backported. -func bodyOrigins(body, repo string) []int { - var origins []int - for _, m := range backportOfRE.FindAllStringSubmatchIndex(body, -1) { - for g := 1; 2*g+1 < len(m); g++ { - start, end := m[2*g], m[2*g+1] - if start < 0 { - continue - } - refs := prRefRE.FindAllStringSubmatch(body[start:end], -1) - if !prRefListEndRE.MatchString(body[end:]) { - refs = refs[:1] - } - for _, ref := range refs { - if ref[1] != "" && !strings.EqualFold(ref[1], repo) { - continue - } - n, _ := strconv.Atoi(ref[2]) - if !slices.Contains(origins, n) { - origins = append(origins, n) - } - } - } - } - return origins -} - -// repoOf is the owner/name of the repository a PR URL, as gh prints it, points -// into. -func repoOf(prURL string) string { - i := strings.LastIndex(prURL, "/pull/") - if i < 0 { - return upstreamRepo - } - parts := strings.Split(prURL[:i], "/") - if len(parts) < 2 { - return upstreamRepo - } - return parts[len(parts)-2] + "/" + parts[len(parts)-1] -} - -// botAuthor reports whether a comment was written by automation rather than by -// a person: the backport bot, a review bot, a dependency bot. gh reports some -// of them without the "[bot]" suffix a GitHub App login otherwise carries, so -// those are named. -func botAuthor(login string) bool { - l := strings.ToLower(login) - if strings.HasSuffix(l, "[bot]") { - return true - } - switch l { - case "github-actions", "coderabbitai", "dependabot", "renovate", "copilot", - "copilot-pull-request-reviewer", "dosubot", "gemini-code-assist": - return true - } - return false -} - -// lastHumanComment is the last comment on a PR not written by the backport bot. -func lastHumanComment(pr backportPR) string { - for i := len(pr.Comments) - 1; i >= 0; i-- { - login := pr.Comments[i].Author.Login - if botAuthor(login) { - continue - } - text := strings.Join(strings.Fields(pr.Comments[i].Body), " ") - if text == "" { - continue - } - if len(text) > 200 { - text = text[:200] - } - return login + ": " + text - } - return "no reason recorded" -} - -// confirmMarker is what a maintainer posts, as the whole of a comment on a -// merged backport PR, to attest that it carries the whole original although -// the audit cannot show that commit by commit, as with a backport squashed into -// one commit. The audit never infers it. -const confirmMarker = "backport-audit: complete" - -// confirmation is that attestation: who wrote it, where, on which backport PR. -type confirmation struct { - By string `json:"by"` - URL string `json:"url"` - BackportPR int `json:"backport_pr"` -} - -// confirmedBy returns the first attestation among prs, or nil. Only a backport -// PR that merged counts, since an open or closed one put nothing on the branch -// to vouch for, and only a comment by someone with a hand in the repository: -// its owner, a member of its organisation or a collaborator. A review bot -// quoting the marker, or a drive-by contributor, confirms nothing. -func confirmedBy(prs []backportPR) *confirmation { - for _, b := range prs { - if b.State != "MERGED" { - continue - } - for _, c := range b.Comments { - if !botAuthor(c.Author.Login) && maintains(c.AuthorAssociation) && markerComment(c.Body) { - return &confirmation{By: c.Author.Login, URL: c.URL, BackportPR: b.Number} - } - } - } - return nil -} - -// markerComment reports whether body is the marker and nothing else: only blank -// lines before it, only whitespace after it, and nothing before it on its own -// line. A comment that holds anything more -- a sentence mentioning it ("do not -// post backport-audit: complete yet"), a quote, a code block -- is talk about -// attesting, not an attestation, and no reading of markdown is needed to tell -// the two apart. Leading spaces or a tab on the marker's line are rejected -// rather than trimmed, since enough of them turn the line into code. -func markerComment(body string) bool { - before, found := strings.CutSuffix(strings.TrimRight(body, " \t\r\n"), confirmMarker) - return found && strings.Trim(before, "\r\n") == "" -} - -// maintains reports whether a comment's GitHub author association is one that -// may vouch for a backport. -func maintains(association string) bool { - switch association { - case "OWNER", "MEMBER", "COLLABORATOR": - return true - } - return false -} - -// commitSource lists the commits a PR contributed: gitSource in the audit, -// fixtures in tests. -type commitSource interface { - // commits lists, oldest first, the commits a PR contributed that a - // backport has to carry. A read that fails is an error, never an empty - // list, since a verdict built on it would be a guess. - commits(mergeCommit string) ([]prCommit, error) -} - -// gitSource reads commits through run, which is git in the audit. -type gitSource struct { - run func(args ...string) (string, error) -} - -// commits reads the PR's side of its merge. main takes PRs as merge commits, so -// that is ^1..^2; a squashed or rebased merge has a single parent, and then the -// commit itself is the change. -// -// Merge commits and commits that change nothing are left out: the backport bot -// drops both (merge_commits: skip, and an empty cherry-pick has nothing to -// apply), so a backport without them is complete, and a "re-trigger CI" commit -// must not make one look partial. -func (g gitSource) commits(mergeCommit string) ([]prCommit, error) { - if mergeCommit == "" { - return nil, errors.New("no merge commit recorded") - } - parents, err := g.run("rev-list", "--parents", "-n", "1", mergeCommit) - if err != nil { - return nil, err - } - var rng string - switch f := strings.Fields(parents); len(f) { - case 2: - rng = mergeCommit + "^.." + mergeCommit - case 3: - rng = mergeCommit + "^1.." + mergeCommit + "^2" - default: - return nil, fmt.Errorf("merge commit %s has %d parents", mergeCommit, len(f)-1) - } - out, err := g.run("log", "--reverse", "--no-merges", "--name-only", "--format=%x00%H%x01%s", rng) - if err != nil { - return nil, err - } - return parsePRCommits(out), nil -} - -// parsePRCommits is the half of gitSource.commits that does not need a -// repository: `git log --name-only` output in, the commits that touched a file -// out. -func parsePRCommits(out string) []prCommit { - var commits []prCommit - for entry := range strings.SplitSeq(out, "\x00") { - head, files, _ := strings.Cut(entry, "\n") - oid, subject, ok := strings.Cut(head, "\x01") - if !ok || strings.TrimSpace(files) == "" { - continue - } - commits = append(commits, prCommit{Oid: oid, Subject: subject}) - } - return commits -} - -// candidateLabel is the backport request under which pr is a candidate for the -// line want, or "" when it is not one. -// -// The request is read against the lines as they stood when the PR merged, even -// for a label added later: nothing the audit lists records when a label was -// added. backport.yaml, which runs again when a label is added, resolves its -// targets at that moment instead, so the two disagree for a label added after -// the line moved on. -func candidateLabel(pr *mainPR, opened []lineOpen, want line) string { - current, previous := targetsAt(opened, pr.MergedAt) - switch { - case pr.labels[labelCurrent] && current != nil && *current == want: - return labelCurrent - case pr.labels[labelPrevious] && previous != nil && *previous == want: - return labelPrevious - } - return "" -} - -func (cfg *config) audit(branch string, cands map[int]*mainPR, opened []lineOpen) (*branchReport, error) { - hist, err := newBranchHistory(cfg.remote, branch) - if err != nil { - return nil, err - } - backports, err := backportPRsFor(branch) - if err != nil { - return nil, err - } - - m := releaseLineRE.FindStringSubmatch(branch) - major, _ := strconv.Atoi(m[1]) - minor, _ := strconv.Atoi(m[2]) - want := line{major, minor} - - numbers := make([]int, 0, len(cands)) - for n := range cands { - numbers = append(numbers, n) - } - sort.Ints(numbers) - - results := []verdict{} - audited := map[int]string{} - for _, n := range numbers { - pr := cands[n] - label := candidateLabel(pr, opened, want) - if label == "" { - continue - } - audited[n] = label - v, err := classify(pr, label, hist, backports[pr.Number], gitSource{run: git}) - if err != nil { - return nil, err - } - results = append(results, v) - } - unlabelled, duplicates := crossCheck(backports, audited, cands) - return &branchReport{Candidates: results, Unlabelled: unlabelled, Duplicates: duplicates}, nil -} - -// crossCheck reads the backport PRs on a branch from the side of the originals -// they claim, which is the only side an unlabelled backport can be seen from. -// audited maps each candidate for the branch to the request it was audited -// under; cands supplies the title and labels of any original that carries a -// backport label at all. -func crossCheck(index map[int][]backportPR, audited map[int]string, cands map[int]*mainPR) ([]unlabelledBackport, []duplicateBackport) { - numbers := make([]int, 0, len(index)) - for n := range index { - numbers = append(numbers, n) - } - sort.Ints(numbers) - - unlabelled := []unlabelledBackport{} - duplicates := []duplicateBackport{} - for _, n := range numbers { - claims := index[n] - linked := make([]linkedPR, 0, len(claims)) - for _, b := range claims { - linked = append(linked, trimPR(b)) - } - sort.Slice(linked, func(i, j int) bool { return linked[i].Number < linked[j].Number }) - - // The URL is derived rather than looked up, so it is there even when the - // title lookup fails; GitHub redirects /pull/N to the issue when N is one. - title, url, labels := "", originURL(claims[0].URL, n), []string{} - if pr, ok := cands[n]; ok { - title, url = pr.Title, pr.URL - for _, req := range backportLabels { - if pr.labels[req.canonical] { - labels = append(labels, req.canonical) - } - } - } - - label, isCandidate := audited[n] - if !isCandidate { - unlabelled = append(unlabelled, unlabelledBackport{ - Number: n, Title: title, URL: url, Labels: labels, BackportPRs: linked, Confirmation: confirmedBy(claims), - }) - } - if kind := duplicateKind(claims); kind != "" { - duplicates = append(duplicates, duplicateBackport{ - Number: n, Title: title, URL: url, Label: label, Kind: kind, BackportPRs: linked, - }) - } - } - return unlabelled, duplicates -} - -// duplicateKind classifies the backport PRs claiming one original, returning -// "" unless more than one of them is live. A closed PR next to an open one is -// the normal shape of a conflicting bot backport redone by hand, and two merged -// ones are history the branch already carries, so neither is flagged. -func duplicateKind(claims []backportPR) string { - open, merged := 0, 0 - for _, b := range claims { - switch b.State { - case "OPEN": - open++ - case "MERGED": - merged++ - } - } - switch { - case open > 0 && merged > 0: - return dupOpenAfterMerge - case open > 1: - return dupOpenTwice - } - return "" -} - -// originURL is the URL of PR n in the repository that sibling, the URL of -// another PR, belongs to. -func originURL(sibling string, n int) string { - i := strings.LastIndex(sibling, "/pull/") - if i < 0 { - return "" - } - return sibling[:i] + "/pull/" + strconv.Itoa(n) -} - -// fillTitles sets the title of every unlabelled or duplicate original that no -// listing carried, with one request covering every branch. -func fillTitles(out map[string]*branchReport) error { - var missing []int - for _, rep := range out { - for _, u := range rep.Unlabelled { - if u.Title == "" && !slices.Contains(missing, u.Number) { - missing = append(missing, u.Number) - } - } - for _, d := range rep.Duplicates { - if d.Title == "" && !slices.Contains(missing, d.Number) { - missing = append(missing, d.Number) - } - } - } - if len(missing) == 0 { - return nil - } - sort.Ints(missing) - titles, err := titlesOf(missing) - for _, rep := range out { - for i := range rep.Unlabelled { - if t, ok := titles[rep.Unlabelled[i].Number]; ok && rep.Unlabelled[i].Title == "" { - rep.Unlabelled[i].Title = t - } - } - for i := range rep.Duplicates { - if t, ok := titles[rep.Duplicates[i].Number]; ok && rep.Duplicates[i].Title == "" { - rep.Duplicates[i].Title = t - } - } - } - return err -} - -// titlesOf fetches the titles of the given issue or PR numbers in a single -// GraphQL request, one alias per number, rather than one gh call each. -// -// gh exits non-zero when any one number fails to resolve -- a reference to a -// deleted PR, say -- but still prints the partial response, so stdout is read -// whatever the exit status and every title that did resolve is kept. -func titlesOf(numbers []int) (map[int]string, error) { - var q strings.Builder - q.WriteString("query($owner:String!,$name:String!){repository(owner:$owner,name:$name){") - for _, n := range numbers { - fmt.Fprintf(&q, "n%d:issueOrPullRequest(number:%d){...on PullRequest{title} ...on Issue{title}}", n, n) - } - q.WriteString("}}") - - stdout, runErr := exec.Command("gh", "api", "graphql", - "-F", "owner={owner}", "-F", "name={repo}", "-f", "query="+q.String()).Output() - if ee, ok := runErr.(*exec.ExitError); ok && len(ee.Stderr) > 0 { - runErr = fmt.Errorf("gh api graphql: %w: %s", runErr, strings.TrimSpace(string(ee.Stderr))) - } - var resp struct { - Data struct { - Repository map[string]*struct { - Title string `json:"title"` - } `json:"repository"` - } `json:"data"` - } - if err := json.Unmarshal(stdout, &resp); err != nil { - if runErr != nil { - return nil, runErr - } - return nil, err - } - titles := map[int]string{} - for alias, node := range resp.Data.Repository { - if n, err := strconv.Atoi(strings.TrimPrefix(alias, "n")); err == nil && node != nil { - titles[n] = node.Title - } - } - if runErr != nil && len(titles) < len(numbers) { - return titles, fmt.Errorf("%d of %d did not resolve: %w", len(numbers)-len(titles), len(numbers), runErr) - } - return titles, nil -} - -// A merged conflict draft can carry only a prefix of the original commits, -// so a merged backport alone settles only a single-commit original. Partial -// delivery still needs the missing commits or a maintainer's attestation. -func classify(pr *mainPR, label string, hist *branchHistory, linked []backportPR, src commitSource) (verdict, error) { - v := verdict{ - Number: pr.Number, - Title: pr.Title, - URL: pr.URL, - Author: pr.Author.Login, - MergedAt: pr.MergedAt.Format(time.RFC3339), - Label: label, - BackportPRs: []linkedPR{}, - } - for _, b := range linked { - v.BackportPRs = append(v.BackportPRs, trimPR(b)) - } - - mergeCommit := pr.MergeCommit.Oid - if hist.contains(mergeCommit) { - v.Status, v.Evidence = statusInBranch, "merged before branch cut" - return v, nil - } - - var merged, open *backportPR - for i := range linked { - switch b := &linked[i]; { - case b.State == "MERGED" && merged == nil: - merged = b - case b.State == "OPEN" && open == nil: - open = b - } - } - openNote := "" - if open != nil { - openNote = fmt.Sprintf("backport PR #%d open", open.Number) - if open.IsDraft { - openNote += " (draft/conflict)" - } - } - prMerged := "" - switch { - case merged != nil: - prMerged = fmt.Sprintf("backport PR #%d merged", merged.Number) - case hist.hasSubject(fmt.Sprintf("[Backport %s] %s", hist.branch, pr.Title)): - prMerged = "bot merge commit on branch" - } - - commits, err := src.commits(mergeCommit) - if err != nil { - return verdict{}, fmt.Errorf("reading the commits of #%d: %w", pr.Number, err) - } - missing := hist.missingFrom(commits, mergeCommit) - evidence := fmt.Sprintf("%d of %s on branch", len(commits)-len(missing), commitCount(len(commits))) - if prMerged != "" { - evidence = prMerged + ", " + evidence - } - - switch { - case len(commits) > 0 && len(missing) == 0: - v.Status, v.Evidence = statusBackported, evidence - return v, nil - case len(commits) <= 1 && prMerged != "": - v.Status, v.Evidence = statusBackported, prMerged - if len(commits) == 1 { - v.Evidence = evidence - } - return v, nil - case len(missing) < len(commits): // some carried, not all - v.Status, v.Evidence, v.MissingCommits = statusPartial, evidence, missing - case prMerged != "": - v.Status, v.Evidence, v.MissingCommits = statusUnverified, evidence, missing - case open != nil: - v.Status, v.Evidence = statusPending, openNote - return v, nil - default: - for _, b := range linked { - if b.State == "CLOSED" { - // A closed backport is usually a deliberate "not needed on this - // line". Carry the reason someone left, so the next release does - // not re-open the same investigation from scratch. - v.Status = statusDropped - v.Evidence = fmt.Sprintf("backport PR #%d closed unmerged", b.Number) - v.Reason = lastHumanComment(b) - return v, nil - } - } - v.Status, v.Evidence = statusMissing, "no backport PR, nothing on branch" - return v, nil - } - - // Partial or unverified from here on. - if openNote != "" { - v.Evidence += "; " + openNote - } - if c := confirmedBy(linked); c != nil { - v.Status, v.Confirmation = statusConfirmed, c - v.Evidence = fmt.Sprintf("confirmed by @%s on backport PR #%d; %s", c.By, c.BackportPR, v.Evidence) - } - return v, nil -} - -func commitCount(n int) string { - if n == 1 { - return "1 commit" - } - return fmt.Sprintf("%d commits", n) -} - -func (cfg *config) report(branch string, rep *branchReport) { - results := rep.Candidates - byStatus := map[string][]verdict{} - for _, r := range results { - byStatus[r.Status] = append(byStatus[r.Status], r) - } - - var counts []string - for _, s := range statusOrder { - if n := len(byStatus[s]); n > 0 { - counts = append(counts, fmt.Sprintf("%s%s=%d%s", cfg.colorFor(s), s, n, cfg.reset)) - } - } - summary := strings.Join(counts, " ") - if summary == "" { - summary = "no candidates" - } - var others []string - if n := len(rep.Duplicates); n > 0 { - others = append(others, fmt.Sprintf("%sDUPLICATE=%d%s", cfg.red, n, cfg.reset)) - } - if n := len(rep.Unlabelled); n > 0 { - others = append(others, fmt.Sprintf("unlabelled=%d", n)) - } - if len(others) > 0 { - summary += " | " + strings.Join(others, " ") - } - fmt.Printf("\n%s=== %s ===%s %d candidate PRs: %s\n", - cfg.bold, branch, cfg.reset, len(results), summary) - - for _, status := range outstanding { - rows := byStatus[status] - if len(rows) == 0 { - continue - } - accent := cfg.colorFor(status) - fmt.Printf("\n %s%s%s (%d):\n", accent, headings[status], cfg.reset, len(rows)) - for _, r := range rows { - cfg.printVerdict(accent, r) - } - } - - if len(rep.Duplicates) > 0 { - fmt.Printf("\n %sDUPLICATE -- one original, more than one live backport PR%s (%d):\n", - cfg.red, cfg.reset, len(rep.Duplicates)) - for _, d := range rep.Duplicates { - label := d.Label - if label == "" { - label = "none for this line" - } - cfg.printOrigin(cfg.red, d.URL, d.Number, d.Title) - fmt.Printf(" %slabel=%s -- %s%s\n", cfg.dim, label, dupDescriptions[d.Kind], cfg.reset) - cfg.printBackportPRs(d.BackportPRs) - } - } - - if rep.clean() { - fmt.Printf(" %snothing outstanding: every candidate is on this branch%s\n", cfg.green, cfg.reset) - } - - if rows := byStatus[statusConfirmed]; len(rows) > 0 { - fmt.Printf("\n %sCONFIRMED -- attested complete by a person where the commits fall short%s (%d, informational):\n", - cfg.green, cfg.reset, len(rows)) - for _, r := range rows { - cfg.printVerdict(cfg.green, r) - fmt.Printf(" %sconfirmation: %s%s\n", cfg.dim, r.Confirmation.URL, cfg.reset) - } - } - - if len(rep.Unlabelled) > 0 { - fmt.Printf("\n %sUNLABELLED -- backport PRs here for originals not labelled for this line%s (%d, informational):\n", - cfg.bold, cfg.reset, len(rep.Unlabelled)) - for _, u := range rep.Unlabelled { - cfg.printOrigin("", u.URL, u.Number, u.Title) - fmt.Printf(" %s%s%s\n", cfg.dim, claimState(u.BackportPRs, u.Confirmation), cfg.reset) - if len(u.Labels) > 0 { - fmt.Printf(" %slabels=%s, which did not target this line at merge%s\n", - cfg.dim, strings.Join(u.Labels, ","), cfg.reset) - } - cfg.printBackportPRs(u.BackportPRs) - } - } -} - -// claimState says what the backport PRs claiming an original have done on the -// branch, so an open claim does not read as a delivered one. A merged one is -// only a merged PR: an unlabelled original is never checked commit by commit, -// so nothing here says it arrived whole unless a person attested it. -func claimState(prs []linkedPR, conf *confirmation) string { - state := "claimed here, closed unmerged" - for _, b := range prs { - switch b.State { - case "MERGED": - if conf != nil { - return "backport PR merged, confirmed by @" + conf.By - } - return "backport PR merged" - case "OPEN": - state = "claimed here, not merged" - } - } - return state -} - -func (cfg *config) printVerdict(accent string, r verdict) { - fmt.Printf(" %s%s%s\n", accent, r.URL, cfg.reset) - fmt.Printf(" %s#%d%s %s\n", cfg.bold, r.Number, cfg.reset, r.Title) - fmt.Printf(" %slabel=%s author=%s merged=%s -- %s%s\n", - cfg.dim, r.Label, r.Author, r.MergedAt[:10], r.Evidence, cfg.reset) - if r.Reason != "" { - fmt.Printf(" %sreason: %s%s\n", cfg.dim, r.Reason, cfg.reset) - } - for _, c := range r.MissingCommits { - fmt.Printf(" %smissing: %.9s %s%s\n", cfg.dim, c.Oid, c.Subject, cfg.reset) - } -} - -func (cfg *config) printOrigin(accent, url string, number int, title string) { - fmt.Printf(" %s%s%s\n", accent, url, cfg.reset) - fmt.Printf(" %s#%d%s %s\n", cfg.bold, number, cfg.reset, title) -} - -func (cfg *config) printBackportPRs(prs []linkedPR) { - for _, b := range prs { - draft := "" - if b.Draft { - draft = " (draft)" - } - fmt.Printf(" %sbackport #%d %s%s %s%s\n", cfg.dim, b.Number, b.State, draft, b.URL, cfg.reset) - } -} diff --git a/cmd/backport-audit/main_test.go b/cmd/backport-audit/main_test.go deleted file mode 100644 index 7d4009437..000000000 --- a/cmd/backport-audit/main_test.go +++ /dev/null @@ -1,1496 +0,0 @@ -/* -Copyright 2026 The Cozystack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "fmt" - "os" - "os/exec" - "slices" - "strings" - "testing" - "time" -) - -func ts(t *testing.T, s string) time.Time { - t.Helper() - when, err := time.Parse(time.RFC3339, s) - if err != nil { - t.Fatalf("bad timestamp %q: %v", s, err) - } - return when -} - -func name(l *line) string { - if l == nil { - return "" - } - return l.branch() -} - -// The real dates, so a change to freezeContractLandedAt or to the era rules -// shows up here rather than on a release cut. -func liveTimeline(t *testing.T) []lineOpen { - t.Helper() - return []lineOpen{ - {line{1, 4}, ts(t, "2026-05-19T13:10:45Z")}, // v1.4.0 published; branch cut earlier, pre-contract - {line{1, 5}, ts(t, "2026-06-22T15:45:38Z")}, // v1.5.0 published - {line{1, 6}, ts(t, "2026-07-22T15:21:48Z")}, // v1.6.0 published - } -} - -func TestTargetsAt(t *testing.T) { - live := liveTimeline(t) - - // A freeze after the cutover opens the line at the rc, not at a stable - // it does not have yet. - frozen17 := append(append([]lineOpen{}, live...), lineOpen{line{1, 7}, ts(t, "2026-09-10T09:00:00Z")}) - - cases := []struct { - name string - opened []lineOpen - merged string - current, previous string - }{ - { - name: "before the cutover, the published-stable rule is in force", - opened: live, merged: "2026-07-01T00:00:00Z", - current: "release-1.5", previous: "release-1.4", - }, - { - name: "after v1.6.0 publishes, still under the old rule", - opened: live, merged: "2026-07-30T00:00:00Z", - current: "release-1.6", previous: "release-1.5", - }, - { - name: "after the cutover with no freeze since, the branch set is unchanged", - opened: live, merged: "2026-09-01T00:00:00Z", - current: "release-1.6", previous: "release-1.5", - }, - { - name: "the instant before a 1.7 freeze still targets 1.6", - opened: frozen17, merged: "2026-09-10T08:59:59Z", - current: "release-1.6", previous: "release-1.5", - }, - { - name: "once release-1.7 exists it is the target, before v1.7.0 ever publishes", - opened: frozen17, merged: "2026-09-10T09:00:01Z", - current: "release-1.7", previous: "release-1.6", - }, - { - name: "only one line exists: no previous", - opened: live[:1], merged: "2026-06-01T00:00:00Z", - current: "release-1.4", previous: "", - }, - { - name: "merged before any line opened: no target at all", - opened: live, merged: "2026-01-01T00:00:00Z", - current: "", previous: "", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - current, previous := targetsAt(tc.opened, ts(t, tc.merged)) - if got := name(current); got != tc.current { - t.Errorf("current = %s, want %s", got, tc.current) - } - if got := name(previous); got != tc.previous { - t.Errorf("previous = %s, want %s", got, tc.previous) - } - }) - } -} - -// A label is read against the lines at merge time however late it was added: -// the audit has no label timestamp. A PR merged on 2026-07-01 and labelled -// after v1.6.0 published is still a release-1.5 candidate, even though -// backport.yaml, running at label time, would have targeted release-1.6. -func TestCandidateLabel(t *testing.T) { - live := liveTimeline(t) - pr := func(merged string, labels ...string) *mainPR { - p := &mainPR{MergedAt: ts(t, merged), labels: map[string]bool{}} - for _, l := range labels { - p.labels[l] = true - } - return p - } - cases := []struct { - name string - pr *mainPR - want map[line]string - }{ - { - name: "merged in the 1.5 era: kind/backport means release-1.5 for good", - pr: pr("2026-07-01T00:00:00Z", labelCurrent), - want: map[line]string{{1, 4}: "", {1, 5}: labelCurrent, {1, 6}: ""}, - }, - { - name: "merged in the 1.5 era: kind/backport-previous means release-1.4 for good", - pr: pr("2026-07-01T00:00:00Z", labelPrevious), - want: map[line]string{{1, 4}: labelPrevious, {1, 5}: "", {1, 6}: ""}, - }, - { - name: "merged in the 1.6 era, both requests", - pr: pr("2026-09-01T00:00:00Z", labelCurrent, labelPrevious), - want: map[line]string{{1, 4}: "", {1, 5}: labelPrevious, {1, 6}: labelCurrent}, - }, - { - name: "no backport label: a candidate nowhere", - pr: pr("2026-09-01T00:00:00Z"), - want: map[line]string{{1, 4}: "", {1, 5}: "", {1, 6}: ""}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - for want, label := range tc.want { - if got := candidateLabel(tc.pr, live, want); got != label { - t.Errorf("%s: candidateLabel = %q, want %q", want.branch(), got, label) - } - } - }) - } -} - -// backport.yaml sorts its branch list numerically descending, so the rank is by -// version and never by when a line opened. The two disagree as soon as a freeze -// overlaps the previous line's stabilisation, and 1.10-versus-1.9 is where a -// lexicographic sort would disagree with both. -func TestTargetsAtRanksByVersionNotByOpenTime(t *testing.T) { - // v1.7.0-rc.1 is cut while v1.6.0 is still unpublished, so the newer line - // opens first in time. - opened := []lineOpen{ - {line{1, 7}, ts(t, "2026-09-10T09:00:00Z")}, - {line{1, 6}, ts(t, "2026-09-20T09:00:00Z")}, - } - current, previous := targetsAt(opened, ts(t, "2026-09-21T00:00:00Z")) - if got := name(current); got != "release-1.7" { - t.Errorf("current = %s, want release-1.7", got) - } - if got := name(previous); got != "release-1.6" { - t.Errorf("previous = %s, want release-1.6", got) - } - - opened = []lineOpen{ - {line{1, 9}, ts(t, "2026-09-10T09:00:00Z")}, - {line{1, 10}, ts(t, "2026-10-10T09:00:00Z")}, - } - current, previous = targetsAt(opened, ts(t, "2026-10-11T00:00:00Z")) - if got := name(current); got != "release-1.10" { - t.Errorf("current = %s, want release-1.10", got) - } - if got := name(previous); got != "release-1.9" { - t.Errorf("previous = %s, want release-1.9", got) - } -} - -func TestParseFreezeDates(t *testing.T) { - // Tag names and dates as `git for-each-ref` prints them. - const out = `v1.4.0 2026-05-19T18:10:45+05:00 -v1.4.0-rc.1 2026-05-13T19:10:58+05:00 -v1.4.0-rc.2 2026-05-14T10:15:38+05:00 -v1.6.0-rc.1 2026-07-07T17:46:06+05:00 -v1.6.1-rc.1 2026-08-04T18:52:49+05:00 -v1.7.0-rc.2 2026-09-12T12:00:00+05:00 -v1.7.0-rc.1 2026-09-10T14:00:00+05:00 -v1.8.0-beta.1 2026-10-01T12:00:00+05:00 -v1.8.0-alpha.1 2026-10-02T12:00:00+05:00 -` - frozen, err := parseFreezeDates(out) - if err != nil { - t.Fatalf("parseFreezeDates: %v", err) - } - - // 1.4 and 1.6 are pre-cutover: their branches were not created by a freeze, - // so the rc tag says nothing about when they appeared. - for _, l := range []line{{1, 4}, {1, 6}} { - if when, ok := frozen[l]; ok { - t.Errorf("%s: pre-cutover rc must not set a freeze date, got %s", l.branch(), when) - } - } - // A patch-line rc is cut from a branch that already exists, and alpha/beta - // do not freeze at all. - for _, l := range []line{{1, 8}} { - if _, ok := frozen[l]; ok { - t.Errorf("%s: alpha/beta must not freeze a line", l.branch()) - } - } - // 1.7 freezes at rc.1, not at whichever rc the listing happens to print first. - want := ts(t, "2026-09-10T14:00:00+05:00") - got, ok := frozen[line{1, 7}] - if !ok { - t.Fatalf("release-1.7: expected a freeze date") - } - if !got.Equal(want) { - t.Errorf("release-1.7 froze at %s, want %s", got, want) - } - if len(frozen) != 1 { - t.Errorf("expected exactly one frozen line, got %d: %v", len(frozen), frozen) - } -} - -// The label listing goes through GitHub search, which stops at 1000 results -// whatever --limit says, so a limit above that must not let a listing cut at -// 1000 pass for a complete one. -func TestCandidateCeiling(t *testing.T) { - cases := []struct { - limit, got int - truncated bool - }{ - {limit: 400, got: 399, truncated: false}, - {limit: 400, got: 400, truncated: true}, - {limit: 1000, got: 999, truncated: false}, - {limit: 1000, got: 1000, truncated: true}, - {limit: 2000, got: 999, truncated: false}, - {limit: 2000, got: 1000, truncated: true}, - } - for _, tc := range cases { - ceiling, remedy := candidateCeiling(tc.limit) - err := truncated(tc.got, ceiling, "the listing", remedy) - if (err != nil) != tc.truncated { - t.Errorf("--limit %d, %d results: truncated = %v, want %v", tc.limit, tc.got, err != nil, tc.truncated) - } - } -} - -// Bodies are the ones real backport PRs carry, trimmed, so a change to the -// grammar is judged against how people actually write them. -func TestBodyOrigins(t *testing.T) { - cases := []struct { - name string - body string - want []int - }{ - { - name: "the plain form", - body: "Backport of #3742 to `release-1.6`.\n\nCherry-picked clean with `-x`.", - want: []int{3742}, - }, - { - name: "two originals in one phrase, and an issue further on that is not one", - body: "Backport of #3938 and #4280 to `release-1.6`, together, because #3938 alone breaks every Kafka deletion (#4276) and #4280 is what fixes that.", - want: []int{3938, 4280}, - }, - { - name: "a list without the serial comma", - body: "Backport of #1, #2 and #3 to `release-1.6`.", - want: []int{1, 2, 3}, - }, - { - name: "a list with the serial comma", - body: "Backport of #1, #2, and #3.", - want: []int{1, 2, 3}, - }, - { - name: "a list ending the line", - body: "Backport of #1, #2 and #3\n\nAll cherry-picked with `-x`.", - want: []int{1, 2, 3}, - }, - { - name: "a list ending the body", - body: "Backport of #1 and #2", - want: []int{1, 2}, - }, - { - name: "a list running on into a clause about its items keeps only the first", - body: "Backport of #10, #20 is not included.", - want: []int{10}, - }, - { - name: "a list running into a parenthesis keeps only the first", - body: "Backport of #10 and #20 (the second only in part).", - want: []int{10}, - }, - { - name: "a to that does not name a release line does not close a list", - body: "Backport of #10, #20 to follow in a separate PR.", - want: []int{10}, - }, - { - name: "an unquoted release line closes a list, whichever line it names", - body: "Backport of #1 and #2 to release-1.5, both clean.", - want: []int{1, 2}, - }, - { - name: "an unquoted release line before a full stop closes a list", - body: "Backport of #1 and #2 to release-1.5.", - want: []int{1, 2}, - }, - { - name: "an unquoted release line ending the body closes a list", - body: "Backport of #1 and #2 to release-1.5", - want: []int{1, 2}, - }, - { - name: "a branch that only starts with a release line does not close a list", - body: "Backport of #10, #20 to release-1.6-fixes will follow later.", - want: []int{10}, - }, - { - name: "a patch version is not a release line", - body: "Backport of #10, #20 to release-1.6.1 later.", - want: []int{10}, - }, - { - name: "a dotted suffix is not a release line", - body: "Backport of #10, #20 to release-1.6.fixes will follow later.", - want: []int{10}, - }, - { - name: "a full stop and a space after the release line close a list", - body: "Backport of #10, #20 to release-1.6. Next, the tests.", - want: []int{10, 20}, - }, - { - name: "a quoted release line running on into a word does not close a list", - body: "Backport of #10, #20 to `release-1.6`-ish branches.", - want: []int{10}, - }, - { - name: "an ampersand is not a separator", - body: "Backport of #1 & #2.", - want: []int{1}, - }, - { - name: "repository-qualified references, alone and in a list", - body: "Backport of cozystack/cozystack#3471 and #3472 to `release-1.6` (clean cherry-pick).", - want: []int{3471, 3472}, - }, - { - name: "this repository's qualifier matches case-insensitively", - body: "Backport of CozyStack/Cozystack#7.", - want: []int{7}, - }, - { - name: "another repository's reference in a list is skipped", - body: "Backport of #10 and other/repo#20.", - want: []int{10}, - }, - { - name: "another repository's reference alone is skipped", - body: "Backport of cozystack/website#20 to `release-1.6`.", - want: nil, - }, - { - name: "another repository's reference first does not hide the rest of a closed list", - body: "Backport of other/repo#20 and #10.", - want: []int{10}, - }, - { - name: "a dependency pulled along with together-with, as #4328 writes it", - body: "Backport of #4253 to `release-1.6`, together with #3460 which it depends on. On 1.6 kube-ovn #99 differs.", - want: []int{4253, 3460}, - }, - { - name: "the same as #4377 writes it", - body: "Backport of #4333 to `release-1.6`, together with #4231 which it depends on. 1.6 has no in-tree chart.", - want: []int{4333, 4231}, - }, - { - name: "the same as #4421 writes it", - body: "## What this PR does\n\nManual backport of #3938 to `release-1.6`, together with #4280.\n\n#3938 cannot go to 1.6 on its own.", - want: []int{3938, 4280}, - }, - { - name: "together-with and an unquoted branch", - body: "Manual backport of #3938 to release-1.6, together with #4280.", - want: []int{3938, 4280}, - }, - { - name: "a closed together-with list keeps every item", - body: "Backport of #10 to `release-1.6`, together with #20 and #30.", - want: []int{10, 20, 30}, - }, - { - name: "a together-with list running on into a clause keeps only its first", - body: "Backport of #10 to release-1.6, together with #20, #30 is tracked separately.", - want: []int{10, 20}, - }, - { - name: "a together-with list running into a to that is not a release line keeps only its first", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to follow later.", - want: []int{10, 20}, - }, - { - name: "a together-with list running into a branch that only starts with a release line keeps only its first", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to release-1.6-fixes later.", - want: []int{10, 20}, - }, - { - name: "a together-with list running into a patch version keeps only its first", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to release-1.6.1 later.", - want: []int{10, 20}, - }, - { - name: "a together-with list running into a dotted suffix keeps only its first", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to release-1.6.fixes later.", - want: []int{10, 20}, - }, - { - name: "a together-with list closed by a release line, a full stop and a space keeps every item", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to release-1.6. Next, the tests.", - want: []int{10, 20, 30}, - }, - { - name: "a together-with list running into a quoted release line and a word keeps only its first", - body: "Backport of #10 to `release-1.6`, together with #20, #30 to `release-1.6`-ish branches.", - want: []int{10, 20}, - }, - { - name: "a together-with list closed by a quoted release line keeps every item", - body: "Backport of #10 to `release-1.6`, together with #20 and #30 to `release-1.5` as well.", - want: []int{10, 20, 30}, - }, - { - name: "together-with in a later sentence is not part of the phrase", - body: "Backport of #7 to `release-1.6`. Together with #8 it fixes the flake.", - want: []int{7}, - }, - { - name: "a hyphenated prefix still anchors", - body: "Hand-backport of #3034 to `release-1.5`. Fixes #12 and #13.", - want: []int{3034}, - }, - { - name: "a list that turns into prose stops at the prose", - body: "Backport of #1, which fixes #2, and #3.", - want: []int{1}, - }, - { - name: "and followed by prose stops at the prose", - body: "Backport of #1 and the follow-up to #2.", - want: []int{1}, - }, - { - name: "every phrase in the body counts, each number once", - body: "Backport of #10 to `release-1.6`.\n\nThe second commit is a backport of #11 and #10.", - want: []int{10, 11}, - }, - { - name: "an article between the phrase and the number is not a reference", - body: "Hand-redone backport of the **kube-ovn-webhook half** of #3997.", - want: nil, - }, - { - name: "no phrase at all", - body: "Fixes #4276 on this branch.", - want: nil, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := bodyOrigins(tc.body, "cozystack/cozystack"); !slices.Equal(got, tc.want) { - t.Errorf("bodyOrigins = %v, want %v", got, tc.want) - } - }) - } -} - -func TestClaimedOrigins(t *testing.T) { - cases := []struct { - name string - pr backportPR - want []int - }{ - { - name: "a hand backport reusing the bot's head naming is counted once", - pr: backportPR{HeadRefName: "backport-3938-to-release-1.6", Body: "Backport of #3938 and #4280 to `release-1.6`."}, - want: []int{3938, 4280}, - }, - { - name: "a head naming another branch says nothing about this one", - pr: backportPR{HeadRefName: "backport-3455-to-release-1.5"}, - want: nil, - }, - { - name: "a head off the bot's pattern leaves only the body", - pr: backportPR{HeadRefName: "backport-3938-release-1.6", Body: "Manual backport of #3938 to `release-1.6`, together with #4280."}, - want: []int{3938, 4280}, - }, - { - name: "qualifiers are read against the repository the PR is in", - pr: backportPR{URL: "https://github.com/example/fork/pull/5", - Body: "Backport of example/fork#3 and cozystack/cozystack#4."}, - want: []int{3}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := claimedOrigins(tc.pr, "release-1.6"); !slices.Equal(got, tc.want) { - t.Errorf("claimedOrigins = %v, want %v", got, tc.want) - } - }) - } -} - -func TestRepoOf(t *testing.T) { - cases := map[string]string{ - "https://github.com/cozystack/cozystack/pull/4431": "cozystack/cozystack", - "https://github.com/example/fork/pull/5": "example/fork", - "": upstreamRepo, - } - for url, want := range cases { - if got := repoOf(url); got != want { - t.Errorf("repoOf(%q) = %q, want %q", url, got, want) - } - } -} - -func bp(n int, state, head, body string) backportPR { - return backportPR{ - Number: n, - State: state, - URL: originURL("https://github.com/cozystack/cozystack/pull/1", n), - HeadRefName: head, - Body: body, - } -} - -func numbersOf[T any](records []T, number func(T) int) []int { - out := []int{} - for _, r := range records { - out = append(out, number(r)) - } - return out -} - -// Modelled on release-1.6: hand backports of main PRs that carry no backport -// label, one of them bundling two originals, next to a closed fork PR that had -// claimed the same pair, and a labelled candidate whose backport merged. -func release16(state4421, state4456 string) []backportPR { - return []backportPR{ - bp(4431, "MERGED", "backport-3455-to-release-1.6", "Backport of #3455 to `release-1.6`."), - bp(4437, "MERGED", "backport-4020-to-release-1.6", "Backport of #4020 to `release-1.6`, without its product changes."), - bp(4456, state4456, "backport-3938-to-release-1.6", - "Backport of #3938 and #4280 to `release-1.6`, together, because #3938 alone breaks every Kafka deletion (#4276)."), - bp(4421, state4421, "backport-3938-release-1.6", - "## What this PR does\n\nManual backport of #3938 to `release-1.6`, together with #4280."), - bp(4474, "MERGED", "backport-4291-to-release-1.6", "Backport of #4291 to `release-1.6`."), - } -} - -func TestCrossCheck(t *testing.T) { - labelled := &mainPR{Number: 4291, Title: "fix(cluster-api): live migration before host eviction", - URL: "https://github.com/cozystack/cozystack/pull/4291", labels: map[string]bool{labelCurrent: true}} - // Labelled, but for another line: it is not audited here, so a backport of - // it on this branch is exactly as unaccounted for as an unlabelled one. - elsewhere := &mainPR{Number: 3455, Title: "fix(e2e): disable guest fsync on ephemeral CI VM disks", - URL: "https://github.com/cozystack/cozystack/pull/3455", labels: map[string]bool{labelPrevious: true}} - cands := map[int]*mainPR{4291: labelled, 3455: elsewhere} - audited := map[int]string{4291: labelCurrent} - - t.Run("merged unlabelled backports are listed, the candidate is not", func(t *testing.T) { - index := indexByOrigin(release16("CLOSED", "MERGED"), "release-1.6") - unlabelled, duplicates := crossCheck(index, audited, cands) - - got := numbersOf(unlabelled, func(u unlabelledBackport) int { return u.Number }) - if want := []int{3455, 3938, 4020, 4280}; !slices.Equal(got, want) { - t.Fatalf("unlabelled originals = %v, want %v", got, want) - } - if len(duplicates) != 0 { - t.Errorf("a closed PR next to a merged one is not a duplicate, got %+v", duplicates) - } - - byNumber := map[int]unlabelledBackport{} - for _, u := range unlabelled { - byNumber[u.Number] = u - } - if u := byNumber[4280]; u.URL != "https://github.com/cozystack/cozystack/pull/4280" || u.Title != "" || len(u.Labels) != 0 { - t.Errorf("#4280: want a derived URL, no title yet and no labels, got %+v", u) - } - if u := byNumber[3455]; u.Title != elsewhere.Title || !slices.Equal(u.Labels, []string{labelPrevious}) { - t.Errorf("#3455: want the candidate's title and its label, got %+v", u) - } - if got := numbersOf(byNumber[3938].BackportPRs, func(l linkedPR) int { return l.Number }); !slices.Equal(got, []int{4421, 4456}) { - t.Errorf("#3938: backport PRs = %v, want both claims sorted", got) - } - }) - - t.Run("two open claims on the same pair are both duplicates", func(t *testing.T) { - index := indexByOrigin(release16("OPEN", "OPEN"), "release-1.6") - _, duplicates := crossCheck(index, audited, cands) - got := numbersOf(duplicates, func(d duplicateBackport) int { return d.Number }) - if want := []int{3938, 4280}; !slices.Equal(got, want) { - t.Fatalf("duplicate originals = %v, want %v", got, want) - } - for _, d := range duplicates { - if d.Kind != dupOpenTwice || d.Label != "" { - t.Errorf("#%d: want kind %s and no label, got %+v", d.Number, dupOpenTwice, d) - } - } - }) - - t.Run("an open claim after a merged one is a leftover", func(t *testing.T) { - index := indexByOrigin(release16("OPEN", "MERGED"), "release-1.6") - _, duplicates := crossCheck(index, audited, cands) - got := numbersOf(duplicates, func(d duplicateBackport) int { return d.Number }) - if want := []int{3938, 4280}; !slices.Equal(got, want) { - t.Fatalf("duplicate originals = %v, want %v", got, want) - } - for _, d := range duplicates { - if d.Kind != dupOpenAfterMerge { - t.Errorf("#%d: want kind %s, got %s", d.Number, dupOpenAfterMerge, d.Kind) - } - } - }) - - t.Run("a labelled candidate claimed twice is a duplicate carrying its label", func(t *testing.T) { - prs := append(release16("CLOSED", "MERGED"), - bp(4480, "OPEN", "fix-migration-again", "Backport of #4291 to `release-1.6`.")) - unlabelled, duplicates := crossCheck(indexByOrigin(prs, "release-1.6"), audited, cands) - if len(duplicates) != 1 || duplicates[0].Number != 4291 || duplicates[0].Label != labelCurrent || - duplicates[0].Kind != dupOpenAfterMerge || duplicates[0].Title != labelled.Title { - t.Fatalf("want one open-after-merge duplicate for #4291 under %s, got %+v", labelCurrent, duplicates) - } - for _, u := range unlabelled { - if u.Number == 4291 { - t.Errorf("an audited candidate must not also be listed as unlabelled") - } - } - }) -} - -func TestDuplicateKind(t *testing.T) { - cases := []struct { - states []string - want string - }{ - {[]string{"OPEN"}, ""}, - {[]string{"MERGED"}, ""}, - {[]string{"CLOSED", "OPEN"}, ""}, - {[]string{"MERGED", "MERGED"}, ""}, - {[]string{"CLOSED", "CLOSED", "MERGED"}, ""}, - {[]string{"OPEN", "OPEN"}, dupOpenTwice}, - {[]string{"OPEN", "CLOSED", "OPEN"}, dupOpenTwice}, - {[]string{"MERGED", "OPEN"}, dupOpenAfterMerge}, - {[]string{"OPEN", "MERGED", "OPEN"}, dupOpenAfterMerge}, - } - for _, tc := range cases { - var claims []backportPR - for i, s := range tc.states { - claims = append(claims, backportPR{Number: i + 1, State: s}) - } - if got := duplicateKind(claims); got != tc.want { - t.Errorf("duplicateKind(%v) = %q, want %q", tc.states, got, tc.want) - } - } -} - -// The gate answers whether everything labelled landed. Unlabelled backports -// can only add to a branch, so they never fail it; a duplicate needs a human -// before the cut, so it always does. -func TestBranchReportClean(t *testing.T) { - landed := []verdict{{Number: 1, Status: statusBackported}, {Number: 2, Status: statusInBranch}} - unlabelled := []unlabelledBackport{{Number: 3455, BackportPRs: []linkedPR{{Number: 4431, State: "OPEN"}}}} - duplicate := []duplicateBackport{{Number: 3938, Kind: dupOpenTwice}} - - cases := []struct { - name string - rep branchReport - want bool - }{ - {"nothing at all", branchReport{}, true}, - {"everything landed", branchReport{Candidates: landed}, true}, - {"only unlabelled backports, even an open one", branchReport{Candidates: landed, Unlabelled: unlabelled}, true}, - {"a duplicate", branchReport{Candidates: landed, Duplicates: duplicate}, false}, - {"a missing candidate", branchReport{Candidates: append(landed, verdict{Number: 3, Status: statusMissing})}, false}, - {"a pending candidate", branchReport{Candidates: append(landed, verdict{Number: 3, Status: statusPending})}, false}, - {"a dropped candidate", branchReport{Candidates: append(landed, verdict{Number: 3, Status: statusDropped})}, false}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := tc.rep.clean(); got != tc.want { - t.Errorf("clean() = %v, want %v", got, tc.want) - } - }) - } -} - -func TestOriginURL(t *testing.T) { - if got := originURL("https://github.com/cozystack/cozystack/pull/4431", 3455); got != "https://github.com/cozystack/cozystack/pull/3455" { - t.Errorf("originURL = %q", got) - } - if got := originURL("not a PR URL", 3455); got != "" { - t.Errorf("originURL of a non-PR URL = %q, want empty", got) - } -} - -// A real listing, of the PR merged in f1f383626: its third commit is an empty -// "re-trigger CI" commit, which the backport bot drops. -const prLogWithEmptyCommit = "\x00fea40d6314424a4105d31c7ee77f818062f5bc1a\x01fix(ci): only overlay current-main images on main-based PRs\n\n" + - ".github/workflows/pull-requests.yaml\nhack/overlay-main-images_test.bats\n" + - "\x002330d6f3ca42306da92190b5e9dddf0bc1054fcd\x01fix(ci): overlay images from the PR base branch, and publish per-line artifacts\n\n" + - ".github/workflows/build-release.yaml\n.github/workflows/pull-requests.yaml\nhack/overlay-main-images_test.bats\n" + - "\x00007d0b1a22ec1aadca95960645d8232a5b202fcb\x01chore(ci): re-trigger CI after a label event produced a no-op run\n" - -func TestParsePRCommits(t *testing.T) { - got := parsePRCommits(prLogWithEmptyCommit) - want := []prCommit{ - {"fea40d6314424a4105d31c7ee77f818062f5bc1a", "fix(ci): only overlay current-main images on main-based PRs"}, - {"2330d6f3ca42306da92190b5e9dddf0bc1054fcd", "fix(ci): overlay images from the PR base branch, and publish per-line artifacts"}, - } - if !slices.Equal(got, want) { - t.Errorf("parsePRCommits = %+v, want %+v", got, want) - } -} - -// fakeSource serves a PR's commits from a fixture, or fails the read. -type fakeSource struct { - commitsOf func(string) []prCommit - err error -} - -func (f fakeSource) commits(mergeCommit string) ([]prCommit, error) { - if f.err != nil { - return nil, f.err - } - return f.commitsOf(mergeCommit), nil -} - -// branchCommit is one commit on a release branch in a test's history. -type branchCommit struct { - oid, subject string - picks []string -} - -// branchOf is release-1.6 holding exactly these commits. -func branchOf(cs ...branchCommit) *branchHistory { - h := &branchHistory{branch: "release-1.6", ref: "origin/release-1.6", - commits: map[string]bool{}, subjects: map[string][]string{}, refs: map[string][]string{}} - for _, c := range cs { - h.commits[c.oid] = true - h.subjects[c.subject] = append(h.subjects[c.subject], c.oid) - if len(c.picks) > 0 { - h.refs[c.oid] = c.picks - } - } - return h -} - -// history is a release branch with one commit per given subject, one per -x -// reference, and the given commits themselves. -func history(subjects, picked, reachable []string) *branchHistory { - var cs []branchCommit - for i, s := range subjects { - cs = append(cs, branchCommit{oid: fmt.Sprintf("s%039d", i), subject: s}) - } - for i, p := range picked { - cs = append(cs, branchCommit{oid: fmt.Sprintf("p%039d", i), subject: "cherry-pick " + p, picks: []string{p}}) - } - for _, r := range reachable { - cs = append(cs, branchCommit{oid: r, subject: "reachable " + r}) - } - return branchOf(cs...) -} - -func comment(login, association, body, url string) prComment { - c := prComment{AuthorAssociation: association, Body: body, URL: url} - c.Author.Login = login - return c -} - -const ( - firstHalf = "1111111111111111111111111111111111111111" - secondHalf = "2222222222222222222222222222222222222222" - emptyOne = "3333333333333333333333333333333333333333" - squashOne = "4444444444444444444444444444444444444444" - mergeOf = "9999999999999999999999999999999999999999" -) - -// A two-commit PR, #4399, plus an empty commit that must never count. -func twoHalves(string) []prCommit { - return parsePRCommits("\x00" + firstHalf + "\x01fix(x): first half\n\nx.go\n" + - "\x00" + secondHalf + "\x01fix(x): second half\n\ny.go\n" + - "\x00" + emptyOne + "\x01chore(ci): re-trigger CI\n") -} - -func pr4399() *mainPR { - pr := &mainPR{Number: 4399, Title: "fix(x): two halves"} - pr.MergeCommit.Oid = mergeOf - return pr -} - -// The bot's conflict drafts stop at the first commit that does not apply and -// drop the rest, so one merged draft reads as a finished backport to anything -// that looks at the PR, or at any single commit, rather than at every commit. -func TestClassifyJudgesEveryCommit(t *testing.T) { - oneCommit := func(string) []prCommit { - return []prCommit{{squashOne, "fix(y): the whole change (#4398)"}} - } - onlyEmpty := func(string) []prCommit { - return parsePRCommits("\x00" + emptyOne + "\x01chore(ci): re-trigger CI\n") - } - bpMerged := backportPR{Number: 4400, State: "MERGED"} - bpOpen := backportPR{Number: 4401, State: "OPEN", IsDraft: true} - botSubject := "[Backport release-1.6] fix(x): two halves" - - cases := []struct { - name string - hist *branchHistory - linked []backportPR - commitsOf func(string) []prCommit - status string - evidence string - missing []string - }{ - { - name: "only the first commit carried, by -x reference: partial", - hist: history(nil, []string{firstHalf}, nil), - commitsOf: twoHalves, - status: statusPartial, evidence: "1 of 2 commits on branch", missing: []string{secondHalf}, - }, - { - name: "both carried, one by -x reference and one by subject; the empty commit is not needed", - hist: history([]string{"fix(x): second half"}, []string{firstHalf}, nil), - commitsOf: twoHalves, - status: statusBackported, evidence: "2 of 2 commits on branch", - }, - { - name: "reworded on the branch, so only an abbreviated -x reference proves it, and the other is reachable", - hist: history(nil, []string{firstHalf[:7]}, []string{secondHalf}), - commitsOf: twoHalves, - status: statusBackported, evidence: "2 of 2 commits on branch", - }, - { - name: "a merged backport PR does not make a partial backport whole", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{bpMerged}, - commitsOf: twoHalves, - status: statusPartial, evidence: "backport PR #4400 merged, 1 of 2 commits on branch", missing: []string{secondHalf}, - }, - { - name: "partial with the rest still open is partial, and says so", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{bpOpen}, - commitsOf: twoHalves, - status: statusPartial, evidence: "1 of 2 commits on branch; backport PR #4401 open (draft/conflict)", missing: []string{secondHalf}, - }, - { - name: "a merged backport PR with none of several commits found proves only that something merged: unverified", - hist: history(nil, nil, nil), - linked: []backportPR{bpMerged}, - commitsOf: twoHalves, - status: statusUnverified, evidence: "backport PR #4400 merged, 0 of 2 commits on branch", missing: []string{firstHalf, secondHalf}, - }, - { - name: "the bot's merge subject alone is no better for several commits", - hist: history([]string{botSubject}, nil, nil), - commitsOf: twoHalves, - status: statusUnverified, evidence: "bot merge commit on branch, 0 of 2 commits on branch", missing: []string{firstHalf, secondHalf}, - }, - { - name: "an -x reference to the merge commit credits no commit by itself", - hist: history(nil, []string{mergeOf[:12]}, nil), - linked: []backportPR{bpMerged}, - commitsOf: twoHalves, - status: statusUnverified, evidence: "backport PR #4400 merged, 0 of 2 commits on branch", missing: []string{firstHalf, secondHalf}, - }, - { - name: "nor does it lift a partial one", - hist: history(nil, []string{mergeOf[:12], firstHalf}, nil), - commitsOf: twoHalves, - status: statusPartial, evidence: "1 of 2 commits on branch", missing: []string{secondHalf}, - }, - { - name: "no commit found and only an open backport PR: pending", - hist: history(nil, nil, nil), - linked: []backportPR{bpOpen}, - commitsOf: twoHalves, - status: statusPending, evidence: "backport PR #4401 open (draft/conflict)", - }, - { - name: "a squashed PR's one commit carried: backported", - hist: history([]string{"fix(y): the whole change (#4398)"}, nil, nil), - commitsOf: oneCommit, - status: statusBackported, evidence: "1 of 1 commit on branch", - }, - { - name: "a squashed PR's commit not found but its backport PR merged: backported", - hist: history(nil, nil, nil), - linked: []backportPR{bpMerged}, - commitsOf: oneCommit, - status: statusBackported, evidence: "backport PR #4400 merged, 0 of 1 commit on branch", - }, - { - name: "and the bot's merge subject settles it the same way", - hist: history([]string{botSubject}, nil, nil), - commitsOf: oneCommit, - status: statusBackported, evidence: "bot merge commit on branch, 0 of 1 commit on branch", - }, - { - name: "nothing a backport has to carry: the merged backport PR decides", - hist: history(nil, nil, nil), - linked: []backportPR{bpMerged}, - commitsOf: onlyEmpty, - status: statusBackported, evidence: "backport PR #4400 merged", - }, - { - name: "nothing anywhere: MISSING", - hist: history(nil, nil, nil), - commitsOf: twoHalves, - status: statusMissing, evidence: "no backport PR, nothing on branch", - }, - { - name: "merged before the cut: in-branch, without reading its commits", - hist: history(nil, nil, []string{mergeOf}), - commitsOf: func(string) []prCommit { - t.Fatalf("commits read for a PR whose merge commit is on the branch") - return nil - }, - status: statusInBranch, evidence: "merged before branch cut", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - v, err := classify(pr4399(), labelCurrent, tc.hist, tc.linked, fakeSource{commitsOf: tc.commitsOf}) - if err != nil { - t.Fatalf("classify: %v", err) - } - if v.Status != tc.status || v.Evidence != tc.evidence { - t.Errorf("got %s %q, want %s %q", v.Status, v.Evidence, tc.status, tc.evidence) - } - var missing []string - for _, c := range v.MissingCommits { - missing = append(missing, c.Oid) - } - if !slices.Equal(missing, tc.missing) { - t.Errorf("missing commits = %v, want %v", missing, tc.missing) - } - // Partial and unverified are outstanding, like MISSING, so they fail - // the gate. - rep := branchReport{Candidates: []verdict{v}} - if want := v.Status == statusBackported || v.Status == statusInBranch; rep.clean() != want { - t.Errorf("clean() = %v for a %s verdict, want %v", rep.clean(), v.Status, want) - } - }) - } -} - -// Only a person's explicit word, on the backport PR that merged, lifts partial -// or unverified; nothing is inferred, and nothing else will do. -func TestClassifyConfirmation(t *testing.T) { - const marker = "backport-audit: complete" - const url = "https://github.com/cozystack/cozystack/pull/4400#issuecomment-1" - onPR := func(state string, comments ...prComment) backportPR { - return backportPR{Number: 4400, State: state, Comments: comments} - } - cases := []struct { - name string - hist *branchHistory - linked []backportPR - status string - by string - }{ - { - name: "partial, attested by a person on the merged backport PR: confirmed", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("coderabbit-reviewer", "NONE", "looks fine", ""), comment("alice", "MEMBER", marker, url))}, - status: statusConfirmed, by: "alice", - }, - { - name: "unverified, attested: confirmed", - hist: history(nil, nil, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "MEMBER", marker, url))}, - status: statusConfirmed, by: "alice", - }, - { - name: "the marker from the backport bot is ignored", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("github-actions", "MEMBER", marker, url))}, - status: statusPartial, - }, - { - name: "also under the bot's [bot] login", - hist: history(nil, nil, nil), - linked: []backportPR{onPR("MERGED", comment("github-actions[bot]", "MEMBER", marker, url))}, - status: statusUnverified, - }, - { - name: "an owner's attestation confirms", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "OWNER", marker, url))}, - status: statusConfirmed, by: "alice", - }, - { - name: "a collaborator's attestation confirms", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "COLLABORATOR", marker, url))}, - status: statusConfirmed, by: "alice", - }, - { - name: "a contributor's marker is ignored", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "CONTRIBUTOR", marker, url))}, - status: statusPartial, - }, - { - name: "a first-time contributor's marker is ignored", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "FIRST_TIME_CONTRIBUTOR", marker, url))}, - status: statusPartial, - }, - { - name: "a marker from someone with no association is ignored", - hist: history(nil, nil, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "NONE", marker, url))}, - status: statusUnverified, - }, - { - name: "a review bot is ignored even as a member with a well-formed marker", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("coderabbitai", "MEMBER", marker, url))}, - status: statusPartial, - }, - { - name: "the marker with an explanation above it attests nothing", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "MEMBER", "Squashed by the contributor; both commits are in.\n\n"+marker, url))}, - status: statusPartial, - }, - { - name: "a member who only mentions the marker in a sentence attests nothing", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "MEMBER", "Do not post backport-audit: complete yet; the second commit is missing.", url))}, - status: statusPartial, - }, - { - name: "a maintainer's attestation counts even after a contributor's", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", - comment("bob", "CONTRIBUTOR", marker, "https://example.invalid/bob"), comment("alice", "MEMBER", marker, url))}, - status: statusConfirmed, by: "alice", - }, - { - name: "the marker on an open backport PR is ignored", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("OPEN", comment("alice", "MEMBER", marker, url))}, - status: statusPartial, - }, - { - name: "the marker on a backport PR closed unmerged is ignored, even next to a merged one", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED"), {Number: 4402, State: "CLOSED", Comments: []prComment{comment("alice", "MEMBER", marker, url)}}}, - status: statusPartial, - }, - { - name: "anything short of the literal marker is not an attestation", - hist: history(nil, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "MEMBER", "backport-audit complete, I think", url))}, - status: statusPartial, - }, - { - name: "complete commit evidence needs no attestation", - hist: history([]string{"fix(x): second half"}, []string{firstHalf}, nil), - linked: []backportPR{onPR("MERGED", comment("alice", "MEMBER", marker, url))}, - status: statusBackported, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - v, err := classify(pr4399(), labelCurrent, tc.hist, tc.linked, fakeSource{commitsOf: twoHalves}) - if err != nil { - t.Fatalf("classify: %v", err) - } - if v.Status != tc.status { - t.Fatalf("status = %s (%s), want %s", v.Status, v.Evidence, tc.status) - } - if tc.by == "" { - if v.Confirmation != nil { - t.Errorf("unexpected confirmation %+v", v.Confirmation) - } - return - } - want := confirmation{By: tc.by, URL: url, BackportPR: 4400} - if v.Confirmation == nil || *v.Confirmation != want { - t.Errorf("confirmation = %+v, want %+v", v.Confirmation, want) - } - if !strings.HasPrefix(v.Evidence, "confirmed by @"+tc.by+" on backport PR #4400; ") { - t.Errorf("evidence %q does not name who confirmed it", v.Evidence) - } - if rep := (branchReport{Candidates: []verdict{v}}); !rep.clean() { - t.Errorf("a confirmed candidate must not fail the gate") - } - if len(v.MissingCommits) == 0 { - t.Errorf("a confirmed candidate keeps the commits nothing on the branch names") - } - }) - } -} - -// A read of the PR's commits that fails must end the audit, not feed a verdict: -// with a merged backport PR linked, falling back to it would say backported on -// the strength of nothing. -func TestClassifyFailedReadIsAnError(t *testing.T) { - linked := []backportPR{{Number: 4400, State: "MERGED"}} - _, err := classify(pr4399(), labelCurrent, history(nil, nil, nil), linked, - fakeSource{err: errors.New("fatal: bad object 9999999")}) - if err == nil || !strings.Contains(err.Error(), "bad object") || !strings.Contains(err.Error(), "#4399") { - t.Fatalf("err = %v, want the failed read of #4399", err) - } -} - -func TestGitSourceCommits(t *testing.T) { - const m = mergeOf - boom := errors.New("fatal: bad object") - cases := []struct { - name string - merge string - parents string - parentErr error - logErr error - wantRange string - wantN int - wantErr bool - }{ - {name: "a merge commit: its second parent's side", merge: m, parents: m + " a b\n", wantRange: m + "^1.." + m + "^2", wantN: 2}, - {name: "a squashed PR: the commit itself", merge: m, parents: m + " a\n", wantRange: m + "^.." + m, wantN: 2}, - {name: "the merge commit is not in the repository", merge: m, parentErr: boom, wantErr: true}, - {name: "the log fails", merge: m, parents: m + " a b\n", logErr: boom, wantRange: m + "^1.." + m + "^2", wantErr: true}, - {name: "an octopus merge is not a PR merge", merge: m, parents: m + " a b c\n", wantErr: true}, - {name: "no merge commit recorded", merge: "", wantErr: true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - gotRange := "" - src := gitSource{run: func(args ...string) (string, error) { - switch args[0] { - case "rev-list": - return tc.parents, tc.parentErr - case "log": - gotRange = args[len(args)-1] - return prLogWithEmptyCommit, tc.logErr - } - return "", fmt.Errorf("unexpected git %v", args) - }} - commits, err := src.commits(tc.merge) - if (err != nil) != tc.wantErr { - t.Fatalf("err = %v, want error %v", err, tc.wantErr) - } - if gotRange != tc.wantRange { - t.Errorf("log range = %q, want %q", gotRange, tc.wantRange) - } - if err != nil && commits != nil { - t.Errorf("a failed read must not return commits, got %v", commits) - } - if !tc.wantErr && len(commits) != tc.wantN { - t.Errorf("got %d commits, want %d", len(commits), tc.wantN) - } - }) - } -} - -// An unlabelled original is never checked commit by commit, so the only thing -// that can make its merged backport read as confirmed is the same attestation. -func TestCrossCheckCarriesConfirmation(t *testing.T) { - attested := bp(4456, "MERGED", "backport-3938-to-release-1.6", "Backport of #3938 to `release-1.6`.") - attested.Comments = []prComment{comment("alice", "MEMBER", "backport-audit: complete", "https://example.invalid/c/1")} - plain := bp(4431, "MERGED", "backport-3455-to-release-1.6", "Backport of #3455 to `release-1.6`.") - unlabelled, _ := crossCheck(indexByOrigin([]backportPR{attested, plain}, "release-1.6"), nil, nil) - got := map[int]*confirmation{} - for _, u := range unlabelled { - got[u.Number] = u.Confirmation - } - if c := got[3938]; c == nil || c.By != "alice" || c.BackportPR != 4456 { - t.Errorf("#3938: confirmation = %+v, want alice on #4456", c) - } - if c := got[3455]; c != nil { - t.Errorf("#3455: confirmation = %+v, want none", c) - } -} - -func TestSameCommit(t *testing.T) { - full := "0123456789abcdef0123456789abcdef01234567" - cases := []struct { - name string - a, b string - want bool - }{ - {"identical", full, full, true}, - {"the reference abbreviates", full, full[:7], true}, - {"the other side abbreviates", full[:7], full, true}, - {"different commits", full, "fff1234000000000000000000000000000000000", false}, - {"an empty name is nobody's commit", full, "", false}, - {"nor is it on the other side", "", full, false}, - } - for _, tc := range cases { - if got := sameCommit(tc.a, tc.b); got != tc.want { - t.Errorf("%s: sameCommit(%q, %q) = %v, want %v", tc.name, tc.a, tc.b, got, tc.want) - } - } -} - -func TestBranchHistoryEvidence(t *testing.T) { - for _, evidence := range []string{"subject", "cherry-pick reference"} { - t.Run(evidence, func(t *testing.T) { - t.Chdir(t.TempDir()) - // Git hooks can export paths into the caller's repository. - for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} { - t.Setenv(key, "") - if err := os.Unsetenv(key); err != nil { - t.Fatal(err) - } - } - t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) - t.Setenv("GIT_CONFIG_NOSYSTEM", "1") - run := func(args ...string) string { - t.Helper() - args = append([]string{"-c", "user.name=Audit Test", "-c", "user.email=audit@example.invalid", "-c", "commit.gpgsign=false"}, args...) - out, err := exec.Command("git", args...).CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - commit := func(file, message string) string { - t.Helper() - if err := os.WriteFile(file, []byte(file+"\n"), 0600); err != nil { - t.Fatal(err) - } - run("add", file) - run("commit", "-m", message) - return run("rev-parse", "HEAD") - } - run("init", "-b", "main") - commit("seed", "chore: seed") - run("switch", "-c", "feature") - first := commit("first", "feat(x): first half") - second := commit("second", "fix tests") - run("switch", "main") - message := "fix tests" - if evidence == "cherry-pick reference" { - message = "chore: unrelated change\n\n(cherry picked from commit " + second + ")" - } - shared := commit("shared", message) - run("branch", "release-1.6") - run("merge", "--no-ff", "feature", "-m", "feat(x): two halves") - merge := run("rev-parse", "HEAD") - run("update-ref", "refs/remotes/origin/main", merge) - run("switch", "release-1.6") - run("cherry-pick", "-x", first) - run("commit", "--amend", "-m", "fix(x): adapted first half\n\n(cherry picked from commit "+first+")") - run("update-ref", "refs/remotes/origin/release-1.6", "HEAD") - - pr := &mainPR{Number: 4399, Title: "feat(x): two halves"} - pr.MergeCommit.Oid = merge - h, err := newBranchHistory("origin", "release-1.6") - if err != nil { - t.Fatal(err) - } - commits := []prCommit{{first, "feat(x): first half"}, {second, "fix tests"}} - if got := h.missingFrom(commits, merge); !slices.Equal(got, commits[1:]) { - t.Errorf("missingFrom = %v, want only the second commit", got) - } - v, err := classify(pr, labelCurrent, h, nil, gitSource{run: git}) - if err != nil { - t.Fatal(err) - } - if v.Status != statusPartial || (&branchReport{Candidates: []verdict{v}}).clean() { - t.Errorf("incomplete backport = %s (%s), want partial and a failing gate", v.Status, v.Evidence) - } - sharedPR := &mainPR{} - sharedPR.MergeCommit.Oid = shared - v, err = classify(sharedPR, labelCurrent, h, nil, gitSource{run: git}) - if err != nil || v.Status != statusInBranch { - t.Errorf("shared history = %s, %v; want in-branch", v.Status, err) - } - - // A branch-only copy without -x must still supply subject evidence. - run("cherry-pick", second) - run("update-ref", "refs/remotes/origin/release-1.6", "HEAD") - h, err = newBranchHistory("origin", "release-1.6") - if err != nil { - t.Fatal(err) - } - v, err = classify(pr, labelCurrent, h, nil, gitSource{run: git}) - if err != nil || v.Status != statusBackported || !(&branchReport{Candidates: []verdict{v}}).clean() { - t.Errorf("complete backport = %s, %v; want backported and a passing gate", v.Status, err) - } - run("update-ref", "-d", "refs/remotes/origin/main") - if _, err := newBranchHistory("origin", "release-1.6"); err == nil { - t.Error("missing main ref must fail the audit") - } - }) - } -} - -// One branch commit cannot supply subject evidence for two original commits. -func TestMissingFrom(t *testing.T) { - const ( - a = "aaaa000000000000000000000000000000000000" - b = "bbbb000000000000000000000000000000000000" - merge = "eeee000000000000000000000000000000000000" - other = "ffff000000000000000000000000000000000000" - ) - pr := []prCommit{{a, "fix tests"}, {b, "fix tests"}} - cases := []struct { - name string - hist *branchHistory - missing []string - }{ - { - name: "one copy of a shared subject carries one commit, not both", - hist: branchOf(branchCommit{oid: "c1", subject: "fix tests"}), - missing: []string{b}, - }, - { - name: "two copies carry both", - hist: branchOf(branchCommit{oid: "c1", subject: "fix tests"}, branchCommit{oid: "c2", subject: "fix tests"}), - missing: nil, - }, - { - name: "a copy that names the first commit with -x is spent on it, not on the second", - hist: branchOf(branchCommit{oid: "c1", subject: "fix tests", picks: []string{a[:9]}}), - missing: []string{b}, - }, - { - name: "the first commit on the branch itself is spent on it, not on the second", - hist: branchOf(branchCommit{oid: a, subject: "fix tests"}), - missing: []string{b}, - }, - { - name: "a copy whose -x names the PR's merge commit can still carry one commit by subject", - hist: branchOf(branchCommit{oid: "c1", subject: "fix tests", picks: []string{merge[:9]}}), - missing: []string{b}, - }, - { - name: "a copy whose -x names some other commit carries nothing here by its subject", - hist: branchOf(branchCommit{oid: "c1", subject: "fix tests", picks: []string{other}}), - missing: []string{a, b}, - }, - { - name: "an -x reference carries its commit whatever the subject", - hist: branchOf(branchCommit{oid: "c1", subject: "reworded", picks: []string{a}}, branchCommit{oid: "c2", subject: "fix tests"}), - missing: nil, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - var got []string - for _, c := range tc.hist.missingFrom(pr, merge) { - got = append(got, c.Oid) - } - if !slices.Equal(got, tc.missing) { - t.Errorf("missing = %v, want %v", got, tc.missing) - } - }) - } -} - -// A shared subject with only one copy on the branch leaves the candidate -// partial, so the gate stays shut. -func TestClassifySharedSubject(t *testing.T) { - pr := pr4399() - commits := func(string) []prCommit { - return []prCommit{{firstHalf, "fix tests"}, {secondHalf, "fix tests"}} - } - v, err := classify(pr, labelCurrent, history([]string{"fix tests"}, nil, nil), nil, fakeSource{commitsOf: commits}) - if err != nil { - t.Fatal(err) - } - if v.Status != statusPartial || v.Evidence != "1 of 2 commits on branch" { - t.Errorf("got %s %q, want partial 1 of 2", v.Status, v.Evidence) - } - if (&branchReport{Candidates: []verdict{v}}).clean() { - t.Errorf("a partial candidate must fail the gate") - } -} - -// Quoted markers, code blocks and surrounding prose do not attest completeness. -func TestMarkerComment(t *testing.T) { - cases := []struct { - name string - body string - want bool - }{ - {"the bare marker", "backport-audit: complete", true}, - {"with blank lines before it and whitespace after it", "\n\nbackport-audit: complete \t\n\n", true}, - {"with CRLF around it", "\r\nbackport-audit: complete\r\n", true}, - {"a sentence that says not to", "Do not post backport-audit: complete yet; the second commit is missing.", false}, - {"mid-sentence", "I think backport-audit: complete applies here.", false}, - {"with words after it", "backport-audit: complete, except the docs", false}, - {"in another case", "Backport-Audit: Complete", false}, - {"quoted", "> backport-audit: complete", false}, - {"quoted, then a reply", "> backport-audit: complete\n\nAgreed?", false}, - {"inside a backtick fence", "```\nbackport-audit: complete\n```", false}, - {"inside a tilde fence", "~~~\nbackport-audit: complete\n~~~", false}, - {"inside nested tilde fences", "~~~~\n~~~\nbackport-audit: complete\n~~~\n~~~~", false}, - {"as an indented code block", "Post this once checked:\n\n backport-audit: complete", false}, - {"indented by a space and a tab after a blank line", "\n\n \tbackport-audit: complete", false}, - {"indented by a tab", "\tbackport-audit: complete", false}, - {"indented by two spaces", " backport-audit: complete", false}, - {"after a line of spaces", " \nbackport-audit: complete", false}, - {"with text glued in front", "xbackport-audit: complete", false}, - {"with an explanation above it", "Checked every commit by hand.\n\nbackport-audit: complete", false}, - {"with a note below it", "backport-audit: complete\n\nthanks!", false}, - {"twice", "backport-audit: complete\nbackport-audit: complete", false}, - {"empty", "", false}, - } - for _, tc := range cases { - if got := markerComment(tc.body); got != tc.want { - t.Errorf("%s: markerComment(%q) = %v, want %v", tc.name, tc.body, got, tc.want) - } - } -} - -func TestBotAuthor(t *testing.T) { - for login, want := range map[string]bool{ - "github-actions": true, - "github-actions[bot]": true, - "coderabbitai": true, - "CodeRabbitAI": true, - "coderabbitai[bot]": true, - "dependabot": true, - "renovate": true, - "copilot": true, - "some-new-app[bot]": true, - "myasnikovdaniil": false, - "alice": false, - "renovate-fan": false, - "bot-enthusiast": false, - } { - if got := botAuthor(login); got != want { - t.Errorf("botAuthor(%q) = %v, want %v", login, got, want) - } - } -} - -func TestClaimState(t *testing.T) { - alice := &confirmation{By: "alice"} - cases := []struct { - states []string - conf *confirmation - want string - }{ - {[]string{"MERGED"}, nil, "backport PR merged"}, - {[]string{"CLOSED", "MERGED"}, nil, "backport PR merged"}, - {[]string{"OPEN", "MERGED"}, nil, "backport PR merged"}, - {[]string{"MERGED"}, alice, "backport PR merged, confirmed by @alice"}, - {[]string{"OPEN"}, nil, "claimed here, not merged"}, - {[]string{"CLOSED", "OPEN"}, nil, "claimed here, not merged"}, - {[]string{"CLOSED"}, nil, "claimed here, closed unmerged"}, - } - for _, tc := range cases { - var prs []linkedPR - for i, s := range tc.states { - prs = append(prs, linkedPR{Number: i + 1, State: s}) - } - if got := claimState(prs, tc.conf); got != tc.want { - t.Errorf("claimState(%v, %v) = %q, want %q", tc.states, tc.conf, got, tc.want) - } - } -} diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md new file mode 100644 index 000000000..b1837f489 --- /dev/null +++ b/cmd/backport-batch/README.md @@ -0,0 +1,128 @@ +# backport-batch + +Builds the automatic backport batch for a release line and gates a patch cut on it. What goes into a batch is decided by the rules of the [backport policy](../../docs/release/backports.md) alone; this page describes how the tool applies them and does not restate them. + +## Usage + +```bash +go build ./cmd/backport-batch +./backport-batch run --line release-1.7 [--onto SHA] [--date YYYY-MM-DD] [--no-workflow-files] [--push] [--policy REF] +./backport-batch run --line release-1.7 --continue [--no-workflow-files] [--push] [--policy REF] +./backport-batch run --line release-1.7 --defer 4321[,4322] --reason "fails the e2e of the app" [--push] [--policy REF] +./backport-batch run --line release-1.7 --drop 4321[,4322] --reason "builds on the 1.8 storage rework" [--push] [--policy REF] +./backport-batch run --line release-1.7 --undrop 4321[,4322] [--push] [--policy REF] +./backport-batch check --line release-1.7 [--cut] [--policy REF] +./backport-batch fold --line release-1.7 --original 4321 --commit SHA [--policy REF] +./backport-batch verify --line release-1.7 [--head SHA] +./backport-batch notify --line release-1.7 [--mention org/team|login] [--push] +./backport-batch report --pr 5246 [--id ID] [--json] +./backport-batch lines [--policy REF] +``` + +[`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) runs `lines`, then `run --continue --push`, `check` and `notify` for each line; [`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs it on schedule as the cozystack-ci App, with `--no-workflow-files`, which it hands to `run`. + +Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Every command but `notify` and `report` fetches `origin` first; `notify` reads only GitHub, and `report` reads GitHub and, for a report that keeps drops, the clone's history. The batch runs only for the lines `lines` prints. + +`run`, `check`, `fold` and `lines` read the `backport-policy` block of the policy page from `origin/main`, or from the ref `--policy` names, which is how a change to the page is tried before it merges; `verify` and `notify` do not read it and take no `--policy`. A line the block lists takes nothing merged into `main` before the day it gives, unless the original carries `backport/include`; any other line starts at its fork point. A page without exactly one readable block stops the run with exit 2. + +Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing. + +### run + +Switches the current working tree to `backport-batch/-`, created at `origin/`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it. + +The pull request body goes to stdout, built from the line's own `.github/PULL_REQUEST_TEMPLATE.md`: the originals on the branch, each with how many commits it brought and which of them differ from the commit they were picked from by `git patch-id`, a resolution or a pick git had to fit, the deferred conflicts with the commit that failed and its files, the skipped pull requests counted by rule with the lists collapsed, the originals' release notes, and two blocks of JSON: the [line's record](#the-lines-record-and-reports) in `backport-batch`, which later runs, `check` and `notify` act on, and the run's report in `backport-batch-report`. The title is `fix(backport): batch for , `. A body that would pass GitHub's 65536-byte limit leaves out its longest skipped lists, and the run prints them on stderr instead; one that what it has to carry still puts over the limit is refused before anything is pushed. + +`--onto SHA` continues a batch already pushed instead of starting a new one. SHA has to be the head the remote has for a batch branch of the line right now, read with `git ls-remote`, so a head someone pushed onto since is refused. What that branch carries stays, and only what it does not carry yet is picked on top, so the next push is a fast-forward. `--continue` takes SHA from the open batch pull request, and starts a new batch when none is open, which is what the scheduled run does. + +`--no-workflow-files` is for a token that may not push a change to `.github/workflows/`, which the scheduled run's App token may not. An original that changes anything there is not picked: the body lists it under "Waiting for a maintainer" with those files, and the `backport-batch` block records it with `"workflows": true`, so `check`, `notify` and a run that continues the batch know it waits. It is never a conflict. A run without the flag, a maintainer's with their own token, picks it like any other original, onto the open batch when it continues one. + +`--defer N[,N] --reason TEXT` takes originals out of the open batch pull request, usually the ones a failing check was pinned on. It builds the batch branch again off `origin/` without their units: while the batch still sits on the line's head, the units before the first one it drops stay as they are, and every later one, a pushed resolution included, is picked again in its own shape, a merge's picks one by one and then the merge with its message. When dropping one leaves a later commit that no longer applies, the run stops and names that commit's original, to defer as well. It adds no new original. It also takes out an original that got `backport/skip` after the batch picked it, which a run never removes on its own and the body lists as still on the branch. The body records each deferred original with the reason in the `backport-batch` block; a run that continues this batch keeps them out, and the next batch tries them again. A batch that carries a merge commit other than an original's, as GitHub's update-branch button makes, is refused, since picking such a merge again would lose what it resolved. When it takes out everything the batch carries, there is nothing left to push, and `--push` closes the batch pull request instead, as [the line's record](#the-lines-record-and-reports) describes. + +`--drop N[,N] --reason TEXT` drops originals from the line: where `--defer` holds one out of this batch, a drop keeps it out of every later batch of the line too, until `--undrop N[,N]` withdraws it. A drop is recorded in the [line's record](#the-lines-record-and-reports), so both need an open batch pull request that carries one, and exit 2 without it; once a batch merges, its drops are carried into the next batch's record. A dropped original the open batch carries is taken off it as `--defer` would, and the rebuild is pushed the same way; any other drop or undrop changes only the record, so the batch stays as it is even when the line moved since. An undropped original is missing until the next run tries it. A drop names the original's merge commit on `main` as well as its number, and a record whose drop does not match what that merge commit merged stops every command with exit 2. An original the line already carries cannot be dropped, and neither can a number that merged into `main` before the line branched. The order is the rules', then the drop: what `backport/skip`, rules 2 to 4 or the line itself decide comes first, with the drop kept on record beside it, and `backport/include` does not lift a drop. The body lists dropped originals under their own heading among the skipped ones, `check` prints a note for each, and `fold` refuses one. + +`--push` pushes the branch, refusing anything but a fast-forward of what the remote has, then opens the draft pull request against the line with `gh pr create --draft`, or updates the open batch pull request with `gh pr edit`, and prints its URL on stderr. On the open batch it first posts the run's report as a comment, so a body never names a report nobody can read; a new batch carries the report in the body it opens with, and the comment follows. Before every write to the pull request it reads the pull request again, and stops with exit 2 once it is merged or closed. When the branch it refuses to push over has no pull request open on it, as a run that stopped between its push and opening the pull request leaves it, the refusal names the `run --onto --push` that continues it. While a batch pull request is open on another branch it refuses, so a line never has two. It never pushes to or edits a batch pull request whose body has no `backport-batch` block either: someone wrote that body by hand, and the batch is theirs to merge or close. After `--defer` alone it replaces the branch, with `--force-with-lease=refs/heads/:`, where head is what `git ls-remote` reads for the branch and has to be the head the rebuild started from: a commit pushed since is never dropped, by the tool's check or, if one lands between that check and the push, by the remote. A record that says the batch is being closed is acted on before anything else, and only by `run --push`. + +### check + +Exits 1 and prints the URL of every pull request the rules take that is neither on the line nor on the open batch pull request, 0 when there is none, and 2 when it cannot tell. A pull request that the [line's record](#the-lines-record-and-reports) lists as deferred, a conflict or one taken out with `--defer`, prints as a warning and does not fail the check: deferring is a maintainer's call. That holds once the batch that recorded it merged or closed too. So does one the open batch lists as waiting for a maintainer because it changes workflows; once that batch is gone, nothing lists it any more, and it is missing. One a maintainer dropped from the line prints as a note, `#N is dropped from on #M ()`. + +`--cut` makes it the gate before a patch release or an rc is cut from a line. A cut tags the line, so a pull request that is only on the open batch pull request fails it too, with one line each: `#N is only on open batch PR #M, merge it before the cut`. One that waits there for a maintainer fails it as well, since nobody decided to leave it out, also when it is all the batch has left: bring it over, or take it out with `run --defer` and a reason. Everything else is as without it, and a cut that passes says `no change the policy takes is missing before the cut`. The scheduled run checks without `--cut`, since its batch is open by design. + +An original with `backport/skip` that the open batch picked before the label came prints a warning in either mode, naming the batch and `run --defer`, which takes it out; it fails neither. + +### fold + +Checks a hand-made resolution of a deferred conflict before it is pushed onto the batch, in the shape the run gives an original. For a merged pull request `--commit` is a merge whose first parent is the head of the open batch pull request and whose message names the original in exactly one `Backported-from:` trailer; its second parent ends a straight line from that head of picks of the pull request's own commits, in their order, each ending in the `(cherry picked from commit …)` line of the commit it came from, none claiming another commit of the same original, a commit of another pull request never among them. For a squash-merged one it is a single commit on that head whose `(cherry picked from commit …)` line names the original's commit on `main`. Either has to change something, and only files the original changes. The [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) gives the commands. fold prints `ok` and exits 0, or prints why not and exits 1. It never commits or pushes; push one checked resolution at a time, since the next one has to sit on the new head. + +### verify + +Holds every unit of a batch to the shape fold checks for one, the clean picks as well as resolutions: it walks the batch's first-parent line from `origin/` to `--head`, or to the head of the open batch pull request, and requires each unit to be one original, merged into `main` after the line branched, in that shape, and no original twice. A commit that carries no original fails it, and so does a merge of the line into the batch. It prints one line per unit, and exits 0 when all pass, 1 when one does not, and 2 when it cannot tell. With `--head` it reads git alone, nothing from GitHub. fold runs only where someone resolved a conflict and chose to run it; [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml) runs verify on every push to a batch pull request, from a build of `main`, with the batch head fetched as data. + +### notify + +Reads the [line's record](#the-lines-record-and-reports) first, as the other commands do, and exits 2 when it cannot read it or when it says the open batch is being closed, with a batch open or not. Then it reads the open batch pull request of the line, trusted as described below, and prints one line for each thing that waits on it: a batch open for three days or more, the shortest gap between two scheduled runs, so one the previous run already saw; the conflicts it deferred; and the originals waiting for a maintainer because they change workflows. Nothing open prints nothing. + +With `--push` it writes the same into one comment on the batch pull request: the conflicts, each to be resolved as the [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. + +The comment starts with `` and is one per pull request: a later run edits it in place rather than adding another, and once nothing waits it says so. Only a comment whose author the batch trusts counts as it, so one someone else wrote with the same first line is neither edited nor believed. It mentions `--mention`, an `org/team` or a login, with or without `@`, only while something waits. A failed batch CI is not among what it reports: the run has no trigger on the batch's CI finishing. + +### report + +Prints a report a batch pull request keeps: the one its body carries, or the one `--id` names. It reads the report comments the batch trusts and falls back to the body, recomputes the report's ID from what it says, and exits 2 when a trusted comment does not match its ID, when a report says anything a run does not write or is for another line than the pull request's, or when the pull request keeps no such report. A report comment counts only when its author is one the batch trusts with a record, so anyone else's comment, whatever ID it carries, is not read. It prints what the classification was made of, whether the pull request's head or body shows the report was published, which it calls unconfirmed when neither does, and one line per original with its category and what the report recorded about it; `--json` prints the report as stored. A report whose comment the run never got to post is printed from the body, with a note that the next `run --push` posts it, or, on a merged or closed batch, that it is kept in the body only. It writes nothing. It reads GitHub, and for a report that keeps drops also the clone's history, without fetching: each drop is bound to `main` as the record's drops are, over the heads of `main` and the line the report was made from, so a drop of an original the report classified has to name that original's merge, and any other a merge of its number on `main`'s first-parent line. A clone without those heads exits 2, asking for a fetch. + +### lines + +Prints the release lines the batch runs for, newest first: of the `release-X.Y` branches on `origin`, the newest that have a stable `vX.Y.Z` tag there, as many as `released-lines` in the policy block says, two when it says nothing, and every branch newer than the newest of them. The tags are read with `git ls-remote`; a tag with a pre-release suffix does not count. `run`, `check` and `fold` refuse any other line, and rule 6 holds each line to the next newer one among them. + +## Rule 4 and the API review gate + +A change to the API is held back only when the [API review gate](../../.github/workflows/api-review-gate.yaml) would call it sizeable. The gate is asked whenever an original changes something it parses: `pkg/apiserver/apiserver.go`, an ApplicationDefinition under `packages/system/*-rd/cozyrds/`, or a `.yaml` or `.yml` file under `packages/` or `internal/` that holds `CustomResourceDefinition` or `APIService` before or after the merge. An edit to the Go types under `api/` alone reaches none of them, so the gate would see no change. The batch calls the gate's own code, `apigate.LoadSnapshot` and `apigate.Classify` from `internal/apigate`, on the trees of the original's first parent and of its merge commit. It reads only what the gate reads, from `git ls-tree` and `git cat-file` into a temporary directory, so no attribute or export rule changes what the gate sees. A reworded description or a new optional field passes; a new or removed group or resource, or a schema change that can reject what the old schema accepted, does not. Migrations, `targetVersion` and CRDs keep their own checks: the gate sees only first-party `cozystack.io` CRDs, and only their sizeable changes, while a vendored operator's CRD is just as unsafe in a patch. + +The CRD check works document by document. In every changed manifest, a YAML, JSON or `.tpl` file or anything under `templates/` or `crds/`, outside `tests/` and `testdata/`, every document that names `CustomResourceDefinition` is read as YAML, which takes JSON too, and the CRDs among them are compared between the parent and the merge by `metadata.name`, compact JSON and quoted keys included. The change counts only when one was added, removed or edited, so an edit to the DaemonSet beside a CRD in one manifest does not, and neither does a sentence or a `lookup` call that merely names the kind. A document that declares `kind: CustomResourceDefinition` on a line of its own but cannot be read as an object, a CRD behind a Helm template guard for one, is compared by its text, so the check fails closed on it, and so does a `List` that holds a CRD. + +## Maintainer overrides + +`run`, `check` and `fold` read the merged pull requests that carry `backport/include` or `backport/skip`, through GitHub search, which answers at most 1000 per label; a listing that long is an error rather than a complete answer. `backport/skip` keeps a pull request out whatever the rules say, and wins over `backport/include`, which lifts rules 2 to 4 and nothing else: what the line already carries and, on the older line, what the newer one does not carry yet still decide. The batch pull request lists every overridden original and what became of it. Adding a label takes triage access to the repository, which is the trust the labels rest on; who added one is not checked. + +## What counts as already there + +An original is on a line when any of these holds, and the batch does not pick it again: + +- its merge commit is reachable from the line; +- picks on the line name its merge commit, or every commit of its own, in the `(cherry picked from commit …)` line git appended to them; +- picking it onto the line would change nothing. + +A pick is known by the line git appends: the last `(cherry picked from commit …)` line of the message's last paragraph, with nothing after it but trailers such as a sign-off. A picked commit keeps its message, and a batch merge carries the original's merge message, so whatever else a message says was written by someone the batch does not answer for: a line in it naming an original, as an `-x` line, a `Backported-from:` trailer or `Backport of #N`, does not count. On commits from before the policy's `batch-only` day it does, since those record the hand backports and earlier batches: + +- a `(cherry picked from commit …)` line anywhere in the trailer block naming its merge commit or, across commits, every commit of its own; +- a `Backported-from:` trailer naming its merge commit; +- `Backport of #N`, which is what a hand backport merged as one commit carries. + +An original is on the open batch pull request when its commits picked there show it the same way. The same rules on the newer line decide rule 6. + +Only the line's own commits count, the ones after the point where it branched from `main`, never a pull request against it: an open backport pull request, its title and its body are anyone's to write. The tool trusts the line's history because only maintainers merge into `release-*`; if that ever changes, a commit that names an original without carrying all of it would hide the rest. The batch pull request counts only when it comes from a `backport-batch/-` branch of this repository, not a fork, and was opened by the bot the scheduled batch runs as, the exact login `BACKPORT_BATCH_BOT` names, `cozystack-ci[bot]` when it is unset and no bot when it is set empty, or by someone whose permission on the repository GitHub reports as write or admin, and only at the head GitHub reports for it. No other bot counts, GitHub Actions' own included, since anything holding a workflow's token writes as it. So a maintainer's run by hand continues the scheduled batch with nothing set, rather than taking its pull request for a look-alike and opening a second one. The notify comment is trusted the same way. A permission GitHub does not answer stops the command with exit 2 rather than count the pull request or the comment out, which would open a second of either. The author association is not enough: an organization member or an invited collaborator can hold read access only. With more than one such pull request open the tool refuses to go on. + +Titles and bodies are read from the merge commits on `main`, which hold them as a maintainer merged them; the live pull request stays editable by its author. + +## The line's record and reports + +The line's record is the `backport-batch` block of its newest batch pull request: the newest pull request against the line from a `backport-batch/-` branch of this repository, opened by an author the batch trusts as above, whether it is open, merged or closed. `run`, `check`, `fold` and `notify` find it in GitHub's list of the line's pull requests, newest first, reading at most ten pages of 100. A batch pull request a person opened without the block is passed over. A block whose author the batch does not trust, a bot's batch without its block, a block that cannot be read whole, a closed record newer than the open batch, or ten pages with no record stop the command with exit 2, naming the pull request, rather than fall back to an older record. + +The record is written only while its pull request is open: a merged or closed batch pull request is never edited or commented on again. What it recorded keeps counting, so a conflict or a deferral a merged batch recorded still prints as a warning. A workflow original it listed as waiting is missing, since no open batch lists it. A run that picks nothing while no batch is open has nowhere to write: it prints the body, says that its classification is not retained, and exits 0. + +The record is only as trustworthy as the body it sits in. Anyone with write access can edit any body, the bot's included, and neither the bot's authorship nor anything in the block proves who edited it last. A hand edit that leaves a readable record is believed; one that breaks it stops every command with exit 2, naming the pull request. A reason or a file name in the record counts only in the form a run writes it, one line with no control, zero-width or bidi character, so text someone typed into the body never reaches a terminal, a body or a comment as they typed it. When the newest batch pull request is deleted and no later one exists, nothing shows it was there: the older record governs, and what was recorded since is lost. + +The record holds only what later commands act on. Every run that classifies the line also makes a report, one line of JSON in a `backport-batch-report` block: every original merged into `main` since the line branched in exactly one category, picked, missing, deferred with what it was deferred for, or skipped with the rule that skipped it, each drop the record keeps with the merge commit it is bound to, its reason and the batch pull request that recorded it, the commits on `main` that merged no pull request, the labels, and what the classification was made of: the heads of `main`, the line and the newer line, the policy, the batch head it describes, the tool's commit, the Actions run and the flags. Its ID is a hash of all that but the time and the run, so two runs that find the same make the same report, and it names no pull request, so a new batch's report is whole before GitHub numbers the pull request. The body carries the latest report beside the record, and a comment on the open batch pull request keeps each one: `` and the same block, folded away. A report already posted is not posted again, and the tool never edits or deletes such a comment, so a report the next run replaces in the body is still read back by `report`, also after the batch merged. A comment that looks like one but whose author the batch does not trust is not read. When a run stops after writing the body but before posting its comment, the next `run --push` on the open batch posts that report first, byte for byte; once the batch is closed nothing is written to it, and the report stays in its body. A run that picks nothing while no batch is open keeps no report. + +A `--defer` that leaves the batch carrying nothing closes it, and writes everything while the pull request is still open: the report comment, then the body, whose record keeps the deferrals and says the batch is closing, with its number, branch and the head the rebuild started from. It then reads the pull request and the branch again, closes the pull request while both are at that head, and deletes the branch with `--force-with-lease` on it, printing `closed `. The next batch tries what was deferred again. A close that stops part way is finished by the next `run --push`, before it does anything else: it closes and deletes when the pull request is open at the frozen head, deletes the branch when the pull request is closed and the branch still at that head, and goes on as usual once both are gone. A run that finishes a close stops there, with exit 0, or with exit 2 when it was asked for `--defer`, `--drop` or `--undrop`, which it does not apply. It never takes a head it did not freeze: with the branch moved, it stops with exit 2 and names both heads. `run` without `--push`, `check` and `fold` stop with exit 2 while the pull request is still open; once it is closed they go on, warning about a branch the close left, and `notify` says nothing. A closed pull request is never written to, so its record says it was closing for good, and a closed pull request whose branch is gone counts as closed. + +One way for the branch to move is a new batch of the same day: it reuses the branch name, and if its push succeeds and opening its pull request fails, the closed batch is still the newest record while its branch now holds the new batch. The next run stops on it, naming the branch, and so does `--onto` with its head. Check what it carries before deleting it with `git push origin --delete backport-batch/-`: the next run picks again what the tool put there from the line, but not a commit someone pushed onto it, such as a resolved conflict. Separate writes to GitHub are not atomic: the re-reads and the leases refuse a race they see, but cannot prevent one. + +## What CI verifies + +The tool does not build or test anything. CI on the batch pull request runs the line's own build and tests against what was picked, and `verify` against its shape, and a maintainer reviews and merges it. A deferred conflict waits for the next batch unless someone resolves it, checks the resolution with `fold` and pushes it, and then CI runs again. + +## Untrusted text + +Titles, release notes and file names in the body come from whoever opened the originals, and a reason from whoever ran `--defer`. A title, a reason or a file name goes into a code span whose fence no backtick inside can close, and the release notes go into the `release-note` block with every run of three backticks or tildes defused, so GitHub acts on no mention, reference, link or HTML in them. Inside that block, in case a note is copied out of it, links lose any target outside this repository, and closing keywords, cross-repository references and mentions have their `@`, `#` and `://` swapped for fullwidth signs, which act on nothing however the backticks around them pair up. The record and the report are one line of JSON each, whose encoding leaves no newline, `<`, `>` or `&` in them, so nothing in a file name or a reason ends their block or opens HTML, and a report comment names no one: numbers are bare, and its only text outside the block is the line, the date and the report's ID. Zero-width, bidi and control characters are dropped first, and everything the tool prints passes the same filter, so nothing it relays can drive a terminal. Commit messages are read through `git cat-file --batch`, which gives each object's length, and end at their first NUL as `git log` shows them, so a message cannot pass for a second commit or hide a trailer from a reader of the log. Every value handed to git comes after `--end-of-options` or `--`, refs pass `git check-ref-format`, SHAs have to be 40 hex digits, and nothing runs through a shell. diff --git a/cmd/backport-batch/apisurface.go b/cmd/backport-batch/apisurface.go new file mode 100644 index 000000000..706e4be62 --- /dev/null +++ b/cmd/backport-batch/apisurface.go @@ -0,0 +1,188 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/cozystack/cozystack/internal/apigate" +) + +// Rule 4's API clause is the API review gate's own verdict: the same +// apigate.LoadSnapshot and apigate.Classify that cmd/api-gate runs in +// api-review-gate.yaml, on the trees of the original's first parent and of its +// merge, so a change the gate passes as additive or descriptive is no reason to +// hold a fix back. + +const ( + apiserverGo = "pkg/apiserver/apiserver.go" + // apigate reads an ApplicationDefinition wherever this matches, whatever + // the file holds. + cozyRDGlob = "packages/system/*-rd/cozyrds/*.yaml" + // apigate skips a YAML file larger than this; reading it would be wasted. + apiFileLimit = 8 << 20 +) + +// apiYAMLRoots hold all YAML apigate.LoadSnapshot parses: the +// ApplicationDefinitions and every first-party CRD or APIService. +var apiYAMLRoots = []string{"packages", "internal"} + +// apiInputs are what apigate.LoadSnapshot reads of a checkout: the YAML roots +// and the apiserver's storage registrations. +var apiInputs = append(slices.Clone(apiYAMLRoots), apiserverGo) + +// apiMarkers are mightBeAPIManifest's: apigate parses no other YAML file that +// holds neither. +var apiMarkers = []string{"-e", "CustomResourceDefinition", "-e", "APIService"} + +var apiSnapshots = map[string]apigate.Snapshot{} + +// apiSnapshot is apigate's snapshot of commit, built from a temporary tree +// holding only what it reads. The blobs come from ls-tree and cat-file rather +// than a checkout or git archive: no attribute filters or export rules apply, +// and a symlink is never followed out of the tree. +func apiSnapshot(commit string) (apigate.Snapshot, error) { + if snap, ok := apiSnapshots[commit]; ok { + return snap, nil + } + out, err := git(append([]string{"ls-tree", "-r", "-z", "-l", "--full-tree", "--end-of-options", commit, "--"}, apiInputs...)...) + if err != nil { + return nil, err + } + paths := map[string]string{} + var blobs []string + for _, entry := range splitNUL(out) { + meta, name, ok := strings.Cut(entry, "\t") + f := strings.Fields(meta) + if !ok || len(f) != 4 { + return nil, fmt.Errorf("ls-tree %.12s: unreadable entry %q", commit, entry) + } + mode, kind, blob := f[0], f[1], f[2] + size, _ := strconv.Atoi(f[3]) + ext := path.Ext(name) + switch { + case kind != "blob" || (mode != "100644" && mode != "100755"): + case name != apiserverGo && ext != ".yaml" && ext != ".yml": + case size > apiFileLimit: + case checkSHA(blob) != nil || !filepath.IsLocal(filepath.FromSlash(name)): + return nil, fmt.Errorf("ls-tree %.12s: unreadable entry %q", commit, entry) + default: + paths[name] = blob + blobs = append(blobs, blob) + } + } + contents, err := readObjects(dedupe(blobs), "blob") + if err != nil { + return nil, err + } + dir, err := os.MkdirTemp("", "backport-batch-api-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + for name, blob := range paths { + file := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(file, contents[blob], 0o600); err != nil { + return nil, err + } + } + snap, err := apigate.LoadSnapshot(dir) + if err != nil { + return nil, fmt.Errorf("the API at %.12s: %w", commit, err) + } + apiSnapshots[commit] = snap + return snap, nil +} + +// readsAPI reports whether the change reaches anything apigate.LoadSnapshot +// parses at either end. A change to anything else leaves both snapshots alike, +// so the gate is not asked. A binary file is searched as well, since apigate +// reads one like any other; a file over its size limit is still asked about, +// which costs a snapshot and changes no verdict. +func readsAPI(parent, merge string, changes []fileChange) (bool, error) { + var yamls []string + for _, c := range changes { + p := c.path + if p == apiserverGo { + return true, nil + } + if ext := path.Ext(p); ext != ".yaml" && ext != ".yml" { + continue + } + if !slices.ContainsFunc(apiYAMLRoots, func(root string) bool { return strings.HasPrefix(p, root+"/") }) { + continue + } + if rd, _ := path.Match(cozyRDGlob, p); rd { + return true, nil + } + yamls = append(yamls, p) + } + for _, rev := range []string{parent, merge} { + found, err := grepFiles(rev, yamls, apiMarkers...) + if err != nil || len(found) > 0 { + return len(found) > 0, err + } + } + return false, nil +} + +// sizeableAPIChange says why the API review gate would call the original's +// change to the API sizeable, or "" when it would not. +func sizeableAPIChange(o *original) (string, error) { + parent, err := resolve(o.merge + "^1") + if err != nil { + return "", err + } + base, err := apiSnapshot(parent) + if err != nil { + return "", err + } + head, err := apiSnapshot(o.merge) + if err != nil { + return "", err + } + findings := apigate.Classify(base, head) + if len(findings) == 0 { + return "", nil + } + why := fmt.Sprintf("%s: %s", findings[0].Category, findings[0].Detail) + if len(findings) > 1 { + why += fmt.Sprintf(", and %d more", len(findings)-1) + } + return why, nil +} + +func dedupe(list []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range list { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go new file mode 100644 index 000000000..7f36ff3f4 --- /dev/null +++ b/cmd/backport-batch/assemble.go @@ -0,0 +1,836 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "os" + "slices" + "strings" + "time" +) + +// pick is the outcome of picking one original onto the batch branch. +type pick struct { + v *verdict + status string // picked, conflict, empty, held out by --defer, or waiting for a maintainer + commit string + files []string + // failed is the commit whose pick conflicted. + failed string + // more counts the conflicting files a recorded conflict no longer names. + more int + reason string +} + +// workflowsDir is where GitHub reads workflows from. A token without the +// workflows permission, a GitHub App's among them, cannot push a commit that +// changes anything under it. +const workflowsDir = ".github/workflows/" + +// workflowFiles lists the workflow files the commits change, each against its +// parent: a push carries every pick, so a change one commit makes and a later +// one takes back is in it although the pull request's merge changes nothing +// there. +func workflowFiles(commits []string) ([]string, error) { + if len(commits) == 0 { + return nil, nil + } + out, err := gitInput(strings.Join(commits, "\n")+"\n", "diff-tree", "--stdin", "--no-commit-id", "--name-only", "-r", "-z", "--no-renames") + if err != nil { + return nil, err + } + var files []string + for _, f := range splitNUL(out) { + if strings.HasPrefix(f, workflowsDir) && !slices.Contains(files, f) { + files = append(files, f) + } + } + return files, nil +} + +func batchBranch(l line, date time.Time) string { + return "backport-batch/" + l.branch() + "-" + date.Format("20060102") +} + +func (a *app) run(args []string) int { + fs := a.policyFlags("run") + lineArg := fs.String("line", "", "release line, release-X.Y") + onto := fs.String("onto", "", "head of the pushed batch branch to continue") + cont := fs.Bool("continue", false, "continue the open batch pull request, or start a batch when none is open") + dateArg := fs.String("date", "", "batch date, YYYY-MM-DD (default: today, UTC)") + deferArg := fs.String("defer", "", "originals to take out of the open batch, N[,N]; rebuilds it without them") + dropArg := fs.String("drop", "", "originals to drop from the line, N[,N]; recorded on the open batch, and taken off it") + undropArg := fs.String("undrop", "", "originals whose drop to withdraw, N[,N]") + noWorkflows := fs.Bool("no-workflow-files", false, "leave an original that changes "+workflowsDir+" to a maintainer, for a token that cannot push one") + reasonArg := fs.String("reason", "", "why --defer takes them out or --drop drops them, recorded in the batch body") + push := fs.Bool("push", false, "push the branch and open or update the draft batch PR") + if err := parse(fs, args); err != nil { + return 2 + } + l, err := parseLine(*lineArg) + if err != nil { + return a.fail(err) + } + date := a.now().UTC() + if *dateArg != "" { + if date, err = time.Parse(time.DateOnly, *dateArg); err != nil { + return a.fail(fmt.Errorf("--date %q is not YYYY-MM-DD", *dateArg)) + } + } + if *onto != "" { + if err := checkSHA(*onto); err != nil { + return a.fail(fmt.Errorf("--onto: %w", err)) + } + } + var deferred, drops []int + reason, ops := "", 0 + for _, op := range []string{*deferArg, *dropArg, *undropArg} { + if op != "" { + ops++ + } + } + switch { + case ops > 1: + return a.fail(errors.New("--defer, --drop and --undrop each change the open batch pull request on their own; give one")) + case *dropArg != "" || *undropArg != "": + if *onto != "" || *cont || *dateArg != "" { + return a.fail(errors.New("--drop and --undrop change the open batch pull request; each takes no --onto, --continue or --date")) + } + if *undropArg != "" && *reasonArg != "" { + return a.fail(errors.New("--undrop takes no --reason")) + } + list := *dropArg + *undropArg + if drops, err = parseNumbers(map[bool]string{true: "--drop", false: "--undrop"}[*dropArg != ""], list); err != nil { + return a.fail(err) + } + if reason = capped(oneLine(clean(*reasonArg, false)), titleLimit); *dropArg != "" && reason == "" { + return a.fail(errors.New("--drop needs a --reason, which the line's record keeps")) + } + case *deferArg != "" || *reasonArg != "": + if deferred, reason, err = parseDefer(*deferArg, *reasonArg); err != nil { + return a.fail(err) + } + if *onto != "" || *cont || *dateArg != "" { + return a.fail(errors.New("--defer rebuilds the open batch pull request; it takes no --onto, --continue or --date")) + } + case *cont && (*onto != "" || *dateArg != ""): + return a.fail(errors.New("--continue reads the batch to continue from the open batch pull request; it takes no --onto or --date")) + } + if err := cleanTree(); err != nil { + return a.fail(err) + } + if err := fetch(); err != nil { + return a.fail(err) + } + b, err := a.prepare(l) + if err != nil { + return a.fail(err) + } + if goOn, code := a.resumeClose(b, *push, deferred != nil || drops != nil); !goOn { + return code + } + if drops != nil { + return a.redrop(b, drops, reason, *undropArg != "", *push) + } + if deferred != nil { + return a.rebuild(b, deferred, reason, *push) + } + if *cont && b.pr != nil { + *onto = b.prHead + } + + branch, start := batchBranch(l, date), b.lineHead + var carried *history + if *onto != "" { + if branch, err = ontoBranch(l, *onto, *dateArg != "", branch); err != nil { + return a.fail(err) + } + if carried, err = readPicks(*onto, b.lineHead, b.textUntil); err != nil { + return a.fail(err) + } + start = *onto + // The batch keeps the date it was opened under, in its title too. + if date, err = branchDate(branch); err != nil { + return a.fail(err) + } + } + if err := checkBranch(branch); err != nil { + return a.fail(err) + } + // Continuing the open batch keeps out what was deferred from it by hand; + // a new batch tries those again. + held := map[int]deferredEntry{} + if carried != nil && b.src.open && b.pr.Head.Ref == branch { + held = b.src.rec.deferred() + } + if err := a.warnReset(branch, start); err != nil { + return a.fail(err) + } + if _, err := git("switch", "--quiet", "--no-track", "-C", branch, "--end-of-options", start); err != nil { + return a.fail(err) + } + + r := newReport(b) + r.opts = runOptions{NoWorkflowFiles: *noWorkflows, Mode: modeNew} + // A drop is a decision about the line, not about one batch: a new batch + // keeps the ones the closed batch recorded. + r.dropped = b.src.rec.dropped() + if carried != nil { + r.opts.Mode = modeContinue + if r.stale, err = keptOut(b, carried); err != nil { + return a.fail(err) + } + } + for _, v := range b.qualifying() { + switch { + case carried != nil && found(carried, v): + r.onBranch = append(r.onBranch, v) + continue + case held[v.o.number].Reason != "": + r.held = append(r.held, pick{v: v, status: "held", reason: held[v.o.number].Reason}) + continue + case v.onBatch != "": + v.rule, v.reason = ruleOnLine, fmt.Sprintf("on the open batch pull request #%d", b.pr.Number) + r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) + continue + } + if *noWorkflows { + files, err := workflowFiles(v.commits) + if err != nil { + return a.fail(err) + } + if len(files) > 0 { + r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: files}) + continue + } + } + p, err := pickOriginal(v) + if err != nil { + return a.fail(err) + } + switch p.status { + case "picked": + r.onBranch = append(r.onBranch, v) + case "conflict": + r.deferred = append(r.deferred, p) + case "empty": + v.rule, v.reason = ruleOnLine, "the earlier picks already carry it" + r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) + } + } + if *onto != "" { + head, err := resolve("HEAD") + if err != nil { + return a.fail(err) + } + descends, err := isAncestor(*onto, head) + if err != nil { + return a.fail(err) + } + if !descends { + return a.fail(fmt.Errorf("%.12s does not descend from --onto %.12s", head, *onto)) + } + } + return a.finish(b, r, branch, date, *push, false) +} + +// keptOut lists the originals with backport/skip, or dropped, that the batch +// still carries: a run never takes a commit off the batch, --defer does. +func keptOut(b *batch, carried *history) ([]*verdict, error) { + var out []*verdict + for _, v := range b.verdicts { + if v.rule != ruleSkipLabel && v.rule != ruleDropped { + continue + } + if err := b.load(v); err != nil { + return nil, err + } + if found(carried, v) { + out = append(out, v) + } + } + return out, nil +} + +func found(h *history, v *verdict) bool { + c, _ := h.found(v.o, v.commits) + return c != "" +} + +// newReport starts the report of a run with what the rules left out and what +// the labels overrode. +func newReport(b *batch) *report { + r := &report{line: b.line, newer: b.newer, skipped: map[int][]*verdict{}, other: len(b.other), start: b.start} + for _, v := range b.verdicts { + if v.rule != 0 { + r.skipped[v.rule] = append(r.skipped[v.rule], v) + } + if v.label != "" { + r.overrides = append(r.overrides, v) + } + } + return r +} + +// finish prints the body and what the run did, and publishes the branch when +// asked to. +func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, rebuilt bool) int { + head, err := resolve("HEAD") + if err != nil { + return a.fail(err) + } + if r.shapes, err = summarize(b, head, r.onBranch); err != nil { + return a.fail(err) + } + if rebuilt && head == b.lineHead && b.pr != nil { + // Deferring everything the batch carries leaves nothing to push: the + // batch is closed, and its record says so first, with the head the + // close is leased on. + r.closing = &closing{PR: b.pr.Number, Branch: branch, Head: b.prHead} + r.opts.Mode = modeClose + } + r.built = builtFrom{Main: b.mainHead, Line: b.lineHead, Newer: b.newerHead, Policy: b.policyAt, Batch: head, Tool: a.tool, + Run: runURL(os.LookupEnv), At: a.now().UTC().Format(time.RFC3339), Options: r.opts} + doc, err := buildReport(b, r) + if err != nil { + return a.fail(err) + } + p := publication{branch: branch, head: head, title: prTitle(b.line, date.Format(time.DateOnly)), report: storedReport{doc: doc, raw: reportJSON(doc)}} + p.record = r.lineRecord(doc) + // A record the next command would refuse is never written. + if err := p.record.validate(b.line); err != nil { + return a.fail(fmt.Errorf("the record this run made %w", err)) + } + r.blocks = recordBlock(p.record) + "\n\n" + reportBlock(p.report.raw) + template, err := lineTemplate(b.lineHead) + if err != nil { + return a.fail(err) + } + body, left := r.body(template) + p.body = body + fmt.Fprint(a.stdout, body) + skipped := 0 + for _, items := range r.skipped { + skipped += len(items) + } + for _, rule := range left { + for _, v := range r.skipped[rule] { + fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) + } + } + for _, p := range r.waiting { + fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) + } + fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", + branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID) + if !push { + return 0 + } + if head == b.lineHead && b.pr == nil { + // Picking nothing is not a failure of the run; check still reports + // whatever the line lacks. Merged and closed batches are never + // written, so there is nowhere to keep what the run found. + fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) + fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) + return 0 + } + if err := a.publish(b, p, rebuilt); err != nil { + return a.fail(err) + } + return 0 +} + +// lineRecord is the record a run leaves for the line: what later commands +// act on, with the full classification left to the report it names. +func (r *report) lineRecord(doc reportDoc) lineRecord { + built := doc.Built + dropped := slices.Clone(r.dropped) + if dropped == nil { + dropped = []droppedEntry{} + } + slices.SortFunc(dropped, func(a, b droppedEntry) int { return a.Original - b.Original }) + return lineRecord{Version: 2, Line: doc.Line, Built: &built, Report: doc.ID, Closing: r.closing, Deferred: slices.Clone(doc.Primary.Deferred), Dropped: dropped, Stale: slices.Clone(doc.Overlays.Stale)} +} + +// branchDate is the date a batch branch was opened under. +func branchDate(branch string) (time.Time, error) { + m := batchBranchRE.FindStringSubmatch(branch) + if m == nil { + return time.Time{}, fmt.Errorf("%q is not a batch branch", branch) + } + return time.Parse("20060102", m[2]) +} + +// cleanTree refuses a working tree with changes or an operation in progress: +// run switches it to the batch branch. +func cleanTree() error { + out, err := git("status", "--porcelain", "--untracked-files=no") + if err != nil { + return err + } + if strings.TrimSpace(out) != "" { + return errors.New("the working tree has changes; run in a clean worktree") + } + for _, name := range []string{"CHERRY_PICK_HEAD", "MERGE_HEAD", "REVERT_HEAD", "REBASE_HEAD", "rebase-merge", "rebase-apply", "sequencer"} { + p, err := git("rev-parse", "--git-path", name) + if err != nil { + return err + } + if _, err := os.Stat(strings.TrimSpace(p)); err == nil { + return fmt.Errorf("an operation is in progress (%s); finish or abort it first", name) + } + } + return nil +} + +// warnReset says, before switch -C moves the local branch to start, which tip +// the move drops commits of and how to get them back: a clean tree says +// nothing about commits only a local branch has. +func (a *app) warnReset(branch, start string) error { + old, err := lookup("refs/heads/" + branch) + if err != nil || old == "" || old == start { + return err + } + kept, err := isAncestor(old, start) + if err != nil || kept { + return err + } + fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old) + return nil +} + +// ontoBranch is the pushed batch branch whose head --onto names, read from the +// remote itself: a head that moved since, or that no batch branch has, is +// refused rather than continued. +func ontoBranch(l line, onto string, dated bool, dateBranch string) (string, error) { + heads, err := remoteBatchHeads(l) + if err != nil { + return "", err + } + var found []string + for branch, sha := range heads { + if sha == onto && (!dated || branch == dateBranch) { + found = append(found, branch) + } + } + switch { + case len(found) == 0 && dated: + return "", fmt.Errorf("--onto %.12s is not the head of %s on %s (%.12s)", onto, dateBranch, remote, heads[dateBranch]) + case len(found) == 0: + return "", fmt.Errorf("--onto %.12s is not the head of any batch branch for %s on %s", onto, l.branch(), remote) + case len(found) > 1: + return "", fmt.Errorf("--onto %.12s heads more than one batch branch; name one with --date", onto) + } + have, err := lookup(onto) + if err != nil { + return "", err + } + if have == "" { + return "", fmt.Errorf("--onto %.12s is not in this repository; fetch it first", onto) + } + return found[0], nil +} + +// remoteBatchHeads lists the batch branches for l on the remote, read with +// ls-remote rather than from the possibly stale remote-tracking refs. +func remoteBatchHeads(l line) (map[string]string, error) { + out, err := git("ls-remote", "--refs", "--heads", "--end-of-options", remote) + if err != nil { + return nil, err + } + heads := map[string]string{} + for ln := range strings.Lines(out) { + sha, ref, ok := strings.Cut(strings.TrimSpace(ln), "\t") + branch, isHead := strings.CutPrefix(ref, "refs/heads/") + if !ok || !isHead || checkSHA(sha) != nil { + continue + } + if m := batchBranchRE.FindStringSubmatch(branch); m != nil && m[1] == l.branch() { + heads[branch] = sha + } + } + return heads, nil +} + +// applyPick runs one cherry-pick onto HEAD and leaves no operation in progress +// behind it: the outcome is picked, conflict, with the conflicting files, or +// empty. +func applyPick(args ...string) (string, []string, error) { + stdout, stderr, code, err := gitStatus(args...) + if err != nil { + return "", nil, err + } + if code == 0 { + return "picked", nil, nil + } + out, err := git("diff", "--name-only", "-z", "--diff-filter=U") + if err != nil { + return "", nil, err + } + if files := splitNUL(out); len(files) > 0 { + _, err := git("cherry-pick", "--abort") + return "conflict", files, err + } + if msg := stdout + stderr; strings.Contains(msg, "is now empty") || strings.Contains(msg, "nothing to commit") { + _, err := git("cherry-pick", "--skip") + return "empty", nil, err + } + failed := fmt.Errorf("failed: %s", strings.TrimSpace(stderr)) + if _, err := git("cherry-pick", "--abort"); err != nil { + return "", nil, errors.Join(failed, err) + } + return "", nil, failed +} + +// lineTemplate is the line's own pull request template, so the batch body has +// the sections reviewers of that line expect. +func lineTemplate(lineHead string) (string, error) { + const template = ".github/PULL_REQUEST_TEMPLATE.md" + out, err := git("--literal-pathspecs", "ls-tree", "--name-only", "--end-of-options", lineHead, "--", template) + if err != nil { + return "", err + } + if strings.TrimSpace(out) == "" { + return "## What this PR does\n\n### Release note\n\n```release-note\n\n```\n", nil + } + return git("show", "--end-of-options", lineHead+":"+template) +} + +// publication is what a run publishes: the branch at head, and the body that +// carries the record and the report. +type publication struct { + branch, head, title, body string + report storedReport + record lineRecord +} + +// publish pushes the batch branch, then opens the draft batch pull request or +// updates the open one. A run pushes only a fast-forward of what the remote +// has. A rebuild replaces the branch, and only the head it was rebuilt from: +// the lease is that head, as ls-remote reads it now, so a commit pushed since +// is never dropped. +// +// The report goes up as a comment before the push and the body that names +// it, so no write leaves a body naming a report nobody can read; a new batch +// carries the report in the body it is opened with, and the comment follows. +// Everything is rendered and measured before the first write. +func (a *app) publish(b *batch, p publication, rebuilt bool) error { + if b.pr != nil && b.pr.Head.Ref != p.branch { + return fmt.Errorf("the open batch pull request #%d is on %s; continue it with --onto %s", b.pr.Number, b.pr.Head.Ref, b.prHead) + } + // A body without the block was written by hand, and the tool's would + // replace it whole, title included; such a batch is the maintainer's. + if b.pr != nil && !b.src.open { + return fmt.Errorf("batch pull request #%d has no %s block, so backport-batch did not write it; merge or close it, and the next run opens a batch of its own", b.pr.Number, deferredFence) + } + if p.head == b.lineHead { + if rebuilt && p.record.Closing != nil { + return a.closeBatch(b, p) + } + return fmt.Errorf("%s carries nothing beyond %s; nothing to push", p.branch, b.line.branch()) + } + heads, err := remoteBatchHeads(b.line) + if err != nil { + return err + } + pushed, onRemote := heads[p.branch] + lease := "" + switch { + case rebuilt: + if !onRemote || pushed != b.prHead { + return fmt.Errorf("%s on %s is at %.12s, not at %.12s, which the rebuild started from; run again", p.branch, remote, pushed, b.prHead) + } + lease = pushed + case onRemote && pushed != p.head: + have, err := lookup(pushed) + if err != nil { + return err + } + descends := false + if have != "" { + if descends, err = isAncestor(pushed, p.head); err != nil { + return err + } + } + if !descends { + refusal := fmt.Sprintf("%s on %s is at %.12s, which %.12s does not descend from; refusing a non-fast-forward push", p.branch, remote, pushed, p.head) + // A run that stopped between its push and opening the pull request + // leaves the branch, and the next run that day builds it anew. + switch c := b.src.carrier; { + case b.pr != nil: + case c != nil && !b.src.open && c.Head.Ref == p.branch: + // Continuing a batch that merged or closed would publish + // what nobody can merge any more. + refusal += fmt.Sprintf("; it is the branch of batch pull request #%d, which is %s; the next run picks again what the tool put on it, but not a commit someone pushed there, such as a resolved conflict: check what %.12s carries, then delete it with git push origin --delete %s and run again", c.Number, prState(c), pushed, p.branch) + default: + refusal += fmt.Sprintf("; no batch pull request is open on it, and backport-batch run --line %s --onto %s --push continues it and opens one", b.line.branch(), pushed) + } + return errors.New(refusal) + } + } + if err := p.fits(); err != nil { + return err + } + if b.pr != nil { + if err := a.keepReports(b, &p); err != nil { + return err + } + } + if pushed != p.head { + if _, err := git(pushArgs(p.branch, p.head, lease)...); err != nil { + return err + } + } + name, err := writeTemp("backport-batch-body-*.md", p.body) + if err != nil { + return err + } + defer os.Remove(name) + if b.pr == nil { + n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name) + if err != nil { + return err + } + fmt.Fprintf(a.stderr, "opened draft %s\n", url) + if err := a.postReport(n, p.report); err != nil { + return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err) + } + return nil + } + if err := a.stillOpen(b.pr.Number); err != nil { + return err + } + if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil { + return err + } + fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) + return nil +} + +// fits says why GitHub would not take the comment or the body. +func (p *publication) fits() error { + if _, err := reportComment(p.report); err != nil { + return err + } + if len(p.body) > bodyLimit { + return fmt.Errorf("the body is %d bytes with only what it cannot leave out, over GitHub's %d; nothing is published", len(p.body), bodyLimit) + } + return nil +} + +// keepReports makes sure the open batch's comments keep both the report its +// body carries now and the one this run made, in that order, each once. +// When this run's report is one already kept, the body carries the kept +// bytes, which differ only in when and in which job it was made. +func (a *app) keepReports(b *batch, p *publication) error { + pr := b.pr + posted, bad, err := findReports(a.gh, pr.Number, b.line) + if err != nil { + return err + } + for _, why := range bad { + a.warn("batch pull request #%d: %s; it does not count as a report", pr.Number, why) + } + earlier, err := bodyReport(pr.Body, b.line) + if err != nil { + return fmt.Errorf("batch pull request #%d: %w", pr.Number, err) + } + kept, ok := posted[p.report.doc.ID] + if !ok && earlier != nil && earlier.doc.ID == p.report.doc.ID { + kept, ok = *earlier, true + } + if ok { + p.body = strings.Replace(p.body, reportBlock(p.report.raw), reportBlock(kept.raw), 1) + p.report = kept + if err := p.fits(); err != nil { + return err + } + } + if earlier != nil { + if _, ok := posted[earlier.doc.ID]; !ok { + if err := a.postReport(pr.Number, *earlier); err != nil { + return fmt.Errorf("posting the earlier report %.12s on #%d: %w", earlier.doc.ID, pr.Number, err) + } + posted[earlier.doc.ID] = *earlier + } + } + if _, ok := posted[p.report.doc.ID]; !ok { + if err := a.postReport(pr.Number, p.report); err != nil { + return fmt.Errorf("posting report %.12s on #%d: %w", p.report.doc.ID, pr.Number, err) + } + } + return nil +} + +// pullURL is where a pull request is on GitHub. +func pullURL(pr *pullRequest) string { + if pullURLRE.MatchString(pr.HTMLURL) { + return pr.HTMLURL + } + return fmt.Sprintf("%spull/%d", ourRepoURL, pr.Number) +} + +// closeBatch closes the open batch that --defer left carrying nothing. Every +// write happens while it is open: the report, then the body whose record +// says it is closing and at which head. The close and the deletion of its +// branch then go only while both are still at that head. +func (a *app) closeBatch(b *batch, p publication) error { + c := *p.record.Closing + if err := p.fits(); err != nil { + return err + } + if err := a.keepReports(b, &p); err != nil { + return err + } + name, err := writeTemp("backport-batch-body-*.md", p.body) + if err != nil { + return err + } + defer os.Remove(name) + if err := a.stillOpen(c.PR); err != nil { + return err + } + if err := a.gh.editPull(c.PR, p.title, name); err != nil { + return fmt.Errorf("recording the close of #%d: %w", c.PR, err) + } + return a.finishClose(b.line, pullURL(b.pr), c) +} + +// finishClose closes the batch pull request a record says is closing, and +// deletes its branch, each only while it is at the head the record froze. +func (a *app) finishClose(l line, url string, c closing) error { + pr, err := a.gh.pull(c.PR) + if err != nil { + return fmt.Errorf("closing #%d, reading it again: %w", c.PR, err) + } + if st := prState(&pr); st != "open" || pr.Head.SHA != c.Head { + return fmt.Errorf("closing #%d: it is %s at %.12s, not open at %.12s, which its record froze", c.PR, st, terminalSafe(pr.Head.SHA), c.Head) + } + heads, err := remoteBatchHeads(l) + if err != nil { + return err + } + if h := heads[c.Branch]; h != c.Head { + return fmt.Errorf("closing #%d: %s on %s is at %.12s, not at %.12s, which its record froze", c.PR, c.Branch, remote, h, c.Head) + } + if err := a.gh.closePull(c.PR); err != nil { + return fmt.Errorf("closing #%d: %w; backport-batch run --line %s --push finishes it", c.PR, err, l.branch()) + } + if err := a.deleteBranch(l, c); err != nil { + return err + } + fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) + return nil +} + +// deleteBranch deletes the branch of a closed batch, leased on the head it +// was closed at: one someone pushed since is a person's to delete or keep. +func (a *app) deleteBranch(l line, c closing) error { + heads, err := remoteBatchHeads(l) + if err != nil { + return err + } + switch h, ok := heads[c.Branch]; { + case !ok: + return nil + case h != c.Head: + return movedBranch(c, h) + } + if _, err := git(deleteArgs(c.Branch, c.Head)...); err != nil { + return fmt.Errorf("deleting %s, left by the close of #%d: %w; backport-batch run --line %s --push deletes it", c.Branch, c.PR, err, l.branch()) + } + return nil +} + +// takenBranch is the refusal for a branch a close left that an open batch +// pull request now uses, whatever it holds, or nil. +func takenBranch(b *batch, c closing) error { + if b.pr == nil || b.pr.Head.Ref != c.Branch { + return nil + } + return fmt.Errorf("batch pull request #%d is open on %s, which the close of #%d left; the branch is that pull request's now, and the close leaves it alone", b.pr.Number, c.Branch, c.PR) +} + +func movedBranch(c closing, h string) error { + return fmt.Errorf("batch pull request #%d was closed at %.12s, but %s is at %.12s now; the next run picks again what the tool put on it, but not a commit someone pushed there, such as a resolved conflict: check what %.12s carries, then delete it with git push origin --delete %s, and run again", c.PR, c.Head, c.Branch, h, h, c.Branch) +} + +// resumeClose finishes a close the record says is under way, before a run +// does anything else, and only under --push. It says whether the run may go +// on: only once the batch is closed and its branch gone. A run that finishes +// one stops there, and fails when it was asked to change the batch. +func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) { + if b.src.rec == nil || b.src.rec.Closing == nil { + return true, 0 + } + c := *b.src.rec.Closing + heads, err := remoteBatchHeads(b.line) + if err != nil { + return false, a.fail(err) + } + h, onRemote := heads[c.Branch] + switch { + case b.src.open && h != c.Head: + return false, a.fail(fmt.Errorf("batch pull request #%d is being closed, but %s is at %.12s, not at %.12s, which its record froze; a person closes it, or pushes the branch back, by hand", c.PR, c.Branch, h, c.Head)) + case b.src.open && !push: + return false, a.fail(fmt.Errorf("batch pull request #%d is being closed: closing it and deleting %s are left; backport-batch run --line %s --push finishes it", c.PR, c.Branch, b.line.branch())) + case b.src.open: + if err := a.finishClose(b.line, pullURL(b.src.carrier), c); err != nil { + return false, a.fail(err) + } + return false, a.finished(c, changing) + case !onRemote: + return true, 0 + case takenBranch(b, c) != nil: + return false, a.fail(takenBranch(b, c)) + case h != c.Head: + return false, a.fail(movedBranch(c, h)) + case !push: + return false, a.fail(fmt.Errorf("batch pull request #%d was closed, but %s is still at %.12s; backport-batch run --line %s --push deletes it", c.PR, c.Branch, h, b.line.branch())) + } + if err := a.deleteBranch(b.line, c); err != nil { + return false, a.fail(err) + } + fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) + return false, a.finished(c, changing) +} + +// finished is the exit of a run that finished a close: a change it was asked +// for needs an open batch pull request, and none is left to make it on. +func (a *app) finished(c closing, changing bool) int { + if !changing { + return 0 + } + return a.fail(fmt.Errorf("finished the close of #%d and made no other change: what this run was asked to change needs an open batch pull request, which the next run opens", c.PR)) +} + +// pushArgs pushes head to branch: as a fast-forward, which the remote checks, +// or, given a lease, over exactly that commit and no other. +func pushArgs(branch, head, lease string) []string { + args := []string{"push", "--quiet"} + if lease != "" { + args = append(args, "--force-with-lease=refs/heads/"+branch+":"+lease) + } + return append(args, "--end-of-options", remote, head+":refs/heads/"+branch) +} + +// deleteArgs deletes branch, only while the remote has it at lease. +func deleteArgs(branch, lease string) []string { + return []string{"push", "--quiet", "--force-with-lease=refs/heads/" + branch + ":" + lease, "--end-of-options", remote, ":refs/heads/" + branch} +} diff --git a/cmd/backport-batch/assemble_test.go b/cmd/backport-batch/assemble_test.go new file mode 100644 index 000000000..31dfd7dda --- /dev/null +++ b/cmd/backport-batch/assemble_test.go @@ -0,0 +1,318 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// A merged pull request lands as its own commits, each picked with -x and +// keeping its author and sign-offs, under a merge commit that carries the +// merge message from main and names it; a squash lands as one pick; a merge +// of main into the pull request's branch is left out; all in main's merge +// order. +func TestRunPicksInMergeOrder(t *testing.T) { + f := newFixture(t) + f.lines() + signoff := "Signed-off" + "-by: " + contributor + mc5 := f.prCommits(5, "fix(app): five", "Body.\n\n```release-note\nfix(app): five is fixed\n```", + change{files: map[string]string{"app/five.txt": "5\n"}, author: contributor, message: "fix(app): start five\n\n" + signoff}, + change{files: map[string]string{"app/five.txt": "5, done\n", "app/five-more.txt": "more\n"}, author: contributor, message: "fix(app): finish five\n\n" + signoff}) + sq6 := f.squash(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "feat(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + mc8 := f.prCommits(8, "fix(app): eight", "", + change{files: map[string]string{"app/eight.txt": "8\n"}}, + change{mergeMain: true}, + change{files: map[string]string{"app/eight.txt": "8, after main\n"}}) + + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.git("branch", "--show-current"); got != batchBranch1 { + t.Fatalf("on %q, want %s", got, batchBranch1) + } + want := []string{"fix(app): five (#5)", "fix(app): six (#6)", "fix(app): eight (#8)"} + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, want) { + t.Errorf("the batch carries %q, want %q", got, want) + } + + line := f.git("rev-parse", "origin/release-1.1") + u5 := f.git("rev-parse", "HEAD~2") + if got := f.git("rev-parse", u5+"^1"); got != line { + t.Errorf("#5's merge sits on %.12s, want the line's head %.12s", got, line) + } + if msg := f.git("log", "-1", "--format=%B", u5); !strings.HasSuffix(msg, "\n\n"+batchTrailer+": "+mc5) || !strings.HasPrefix(msg, "fix(app): five (#5)") { + t.Errorf("#5's merge does not carry the merge message and name %s:\n%s", mc5, msg) + } + own := strings.Fields(f.git("rev-list", "--reverse", mc5+"^1.."+mc5+"^2")) + picks := strings.Fields(f.git("rev-list", "--reverse", u5+"^2", "^"+u5+"^1")) + if len(picks) != 2 || len(own) != 2 { + t.Fatalf("#5 brought %d picks of its %d commits, want 2", len(picks), len(own)) + } + for i, c := range picks { + msg := f.git("log", "-1", "--format=%B", c) + if got := f.git("log", "-1", "--format=%an <%ae>", c); got != contributor { + t.Errorf("pick %d is authored by %q, want %q", i, got, contributor) + } + if !strings.Contains(msg, signoff+"\n(cherry picked from commit "+own[i]+")\nSigned-off-by: Batch Test ") { + t.Errorf("pick %d lacks the contributor's sign-off, its -x line for %s or the batch's sign-off:\n%s", i, own[i], msg) + } + } + + u6 := f.git("rev-parse", "HEAD~1") + if parents := strings.Fields(f.git("rev-list", "--parents", "-n", "1", u6)); len(parents) != 2 || parents[1] != u5 { + t.Errorf("#6 is %v, want one commit on #5's merge", parents) + } + if msg := f.git("log", "-1", "--format=%B", u6); !strings.Contains(msg, "(cherry picked from commit "+sq6+")") { + t.Errorf("#6's pick lacks its -x line:\n%s", msg) + } + + picks8 := strings.Fields(f.git("rev-list", "--reverse", "HEAD^2", "^HEAD^1")) + if got := len(picks8); got != 2 { + t.Errorf("#8 brought %d commits, want its own 2 without the merge of main", got) + } + if !strings.HasSuffix(f.git("log", "-1", "--format=%B", "HEAD"), batchTrailer+": "+mc8) { + t.Error("#8's merge does not name its merge commit") + } + if f.git("rev-parse", "HEAD:app/eight.txt") != f.git("rev-parse", mc8+":app/eight.txt") { + t.Error("app/eight.txt is not what main has after #8") + } + for _, s := range []string{"[#5]", "[#6]", "[#8]", "fix(app): five is fixed", "Rule 2", "`fix(app): five`, 2 commits", "`fix(app): six`, 1 commit"} { + if !strings.Contains(stdout, s) { + t.Errorf("the body lacks %q:\n%s", s, stdout) + } + } + if code, out, stderr := f.exec(nil, "verify", "--line", "release-1.1", "--head", f.git("rev-parse", "HEAD")); code != 0 { + t.Errorf("verify of the batch = %d\n%s\n%s", code, out, stderr) + } +} + +// A conflict is aborted, recorded with its files, and the next original is +// still picked. +func TestRunRecordsConflictsAndGoesOn(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(chart): a line-only value", map[string]string{"chart/values.yaml": "replicas: 3\nimage: app:1\n"}) + f.pr(5, "fix(chart): five", "", map[string]string{"chart/values.yaml": "replicas: 5\nimage: app:1\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q, want only #6", got) + } + if out := f.git("status", "--porcelain"); out != "" { + t.Errorf("the tree is left dirty:\n%s", out) + } + if _, err := os.Stat(filepath.Join(f.dir, ".git", "CHERRY_PICK_HEAD")); err == nil { + t.Error("a cherry-pick is left in progress") + } + l, _ := parseLine("release-1.1") + deferred, err := parseDeferred(stdout, l) + if err != nil || len(deferred[5].Files) == 0 || len(deferred) != 1 { + t.Errorf("deferred = %v, %v; want #5", deferred, err) + } + if !strings.Contains(stdout, "### Deferred conflicts") || !strings.Contains(stdout, "`chart/values.yaml`") { + t.Errorf("the body does not name the conflict and its file:\n%s", stdout) + } +} + +// An original the earlier picks already carry is skipped as an empty pick. +func TestRunSkipsAnEmptyPick(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): add x", "", map[string]string{"app/x.txt": "x\n"}) + f.pr(6, "revert(app): drop x", "", map[string]string{"app/x.txt": ""}) + f.pr(7, "fix(app): add x again", "", map[string]string{"app/x.txt": "x\n"}) + + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): add x (#5)"}) { + t.Errorf("picked %q, want only #5", got) + } + if !strings.Contains(stdout, "the earlier picks already carry it") { + t.Errorf("#7 is not reported as carried by the earlier picks:\n%s", stdout) + } +} + +// A hook in the repository or the user's configuration never rewrites what a +// pick carries. +func TestRunIgnoresHooks(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + hooks := filepath.Join(f.dir, ".git", "hooks") + for _, hook := range []string{"prepare-commit-msg", "commit-msg"} { + if err := os.WriteFile(filepath.Join(hooks, hook), []byte("#!/bin/sh\necho rewritten by a hook > \"$1\"\n"), 0o755); err != nil { + t.Fatal(err) + } + } + if code, _, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): five (#5)"}) { + t.Errorf("picked %q, want #5 as it was merged", got) + } +} + +func TestRunRefusesADirtyTree(t *testing.T) { + f := newFixture(t) + f.lines() + f.write("README.md", "changed\n") + code, _, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 2 || !strings.Contains(stderr, "has changes") { + t.Errorf("run on a dirty tree = %d (%s), want 2", code, stderr) + } + if f.git("branch", "--show-current") != "main" { + t.Error("the tree was switched anyway") + } + + // A pick that stopped empty leaves the tree clean and the pick going. + f.git("checkout", "-q", "--", "README.md") + if err := os.WriteFile(filepath.Join(f.dir, ".git", "CHERRY_PICK_HEAD"), []byte(f.git("rev-parse", "HEAD")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + code, _, stderr = f.exec(nil, "run", "--line", "release-1.1") + if code != 2 || !strings.Contains(stderr, "operation is in progress (CHERRY_PICK_HEAD)") { + t.Errorf("run during a cherry-pick = %d (%s), want 2", code, stderr) + } +} + +// A clean tree says nothing about commits only the local batch branch has, so +// a run or a rebuild that resets that branch over them names its tip and the +// command that gets them back; one that drops nothing says nothing. +func TestResetNamesWhatItDrops(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + unpushed := func(base string) string { + t.Helper() + f.git("switch", "-q", "-C", batchBranch1, base) + f.write("app/mine.txt", "unpushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "an unpushed resolution") + tip := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + return tip + } + notice := func(stderr, tip string) bool { + t.Helper() + restore := "git branch " + batchBranch1 + "-" + tip[:12] + " " + tip + if !strings.Contains(stderr, "resetting the local "+batchBranch1+" from "+tip+",") || !strings.Contains(stderr, restore) { + return false + } + f.git(strings.Fields(restore)[1:]...) + if got := f.git("rev-parse", batchBranch1+"-"+tip[:12]); got != tip { + t.Errorf("%s restored %.12s, want %.12s", restore, got, tip) + } + return true + } + + tip := unpushed("origin/release-1.1") + if code, _, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !notice(stderr, tip) { + t.Errorf("run over an unpushed commit = %d, want it named with its restore:\n%s", code, stderr) + } + f.git("switch", "-q", "main") + f.git("branch", "-f", batchBranch1, "origin/release-1.1~1") + if code, _, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stderr, "resetting") { + t.Errorf("run over a branch it fast-forwards = %d, want no notice:\n%s", code, stderr) + } + + f.git("switch", "-q", "main") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + tip = unpushed(head) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the app e2e"); code != 0 || !notice(stderr, tip) { + t.Errorf("a rebuild over an unpushed commit = %d, want it named with its restore:\n%s", code, stderr) + } +} + +// --push pushes a fresh branch and opens the draft, and a second push of the +// same branch updates the open pull request instead. +func TestRunPush(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + if !strings.Contains(stderr, "opened draft https://github.com/cozystack/cozystack/pull/90\n") { + t.Errorf("the run does not print the pull request it opened:\n%s", stderr) + } + l, _ := parseLine("release-1.1") + if _, err := parseDeferred(stdout, l); err != nil || !strings.HasPrefix(stdout, "## What this PR does") || strings.Contains(stdout, "https://github.com/cozystack/cozystack/pull/90") { + t.Errorf("stdout is no longer the body alone (%v):\n%s", err, stdout) + } + head := f.git("rev-parse", "HEAD") + if got := f.remoteHead(batchBranch1); got != head { + t.Errorf("remote %s = %q, want %s", batchBranch1, got, head) + } + if want := []string{"release-1.1 " + batchBranch1 + " fix(backport): batch for release-1.1, 2026-10-26"}; !slices.Equal(gh.created, want) { + t.Errorf("created %q, want %q", gh.created, want) + } + + f.git("switch", "-q", "main") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, gh.bodies[0])} + code, _, stderr = f.exec(gh, "run", "--line", "release-1.1", "--onto", head, "--push") + if code != 0 { + t.Fatalf("run --onto --push = %d\n%s", code, stderr) + } + if !strings.Contains(stderr, "updated https://github.com/cozystack/cozystack/pull/90\n") { + t.Errorf("the run does not print the pull request it updated:\n%s", stderr) + } + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): five (#5)", "fix(app): six (#6)"}) { + t.Errorf("the pushed batch carries %q, want #5 then #6", got) + } + if len(gh.edited) != 1 || gh.edited[0] != "#90 fix(backport): batch for release-1.1, 2026-10-26" { + t.Errorf("edited %q, want #90", gh.edited) + } + if body := gh.bodies[len(gh.bodies)-1]; !strings.Contains(body, "[#5]") || !strings.Contains(body, "[#6]") { + t.Errorf("the updated body does not list both originals:\n%s", body) + } +} + +// With a batch pull request open on another branch, a fresh push would open +// a second one; it is refused before anything is pushed. +func TestRunPushRefusesASecondBatch(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + old := "backport-batch/release-1.1-20261022" + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+old) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{prs: []pullRequest{f.batchPR(90, "release-1.1", old, "")}} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 2 || !strings.Contains(stderr, "continue it with --onto") { + t.Errorf("run --push = %d (%s), want a refusal", code, stderr) + } + if f.remoteHead(batchBranch1) != "" || len(gh.created)+len(gh.edited) != 0 { + t.Error("something was pushed or opened") + } +} diff --git a/cmd/backport-batch/batchbot_test.go b/cmd/backport-batch/batchbot_test.go new file mode 100644 index 000000000..ff1cb342a --- /dev/null +++ b/cmd/backport-batch/batchbot_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "slices" + "strings" + "testing" +) + +// A maintainer's run by hand continues the batch pull request the scheduled +// run opened as its bot, with nothing set: it edits that pull request rather +// than opening a second one for the line. A record another bot wrote is no +// record the batch trusts, and the run stops rather than reach past it. +func TestHandRunContinuesTheBotsBatch(t *testing.T) { + for _, tc := range []struct { + name, author string + continued bool + }{ + {"the scheduled run's bot", "cozystack-ci[bot]", true}, + {"another bot", "renovate[bot]", false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{bot: batchBotLogin(func(string) (string, bool) { return "", false })} + openBatch(t, f, gh) + gh.prs[0].User.Login, gh.prs[0].User.Type = tc.author, "Bot" + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.now = f.now.AddDate(0, 0, 3) + + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if tc.continued && code != 0 || !tc.continued && (code != 2 || !strings.Contains(stderr, "batch pull request #90 (open) carries a ```backport-batch block but was opened by \"renovate[bot]\"")) { + t.Fatalf("run --continue = %d\n%s", code, stderr) + } + continued := slices.Equal(gh.edited, []string{"#90 fix(backport): batch for release-1.1, 2026-10-26"}) && len(gh.created) == 1 + if continued != tc.continued || len(gh.created) != 1 { + t.Errorf("edited %q, created %q; want the batch of %s continued=%v and no second one\n%s", gh.edited, gh.created, tc.author, tc.continued, stderr) + } + if !tc.continued && !strings.Contains(stderr, "ignoring #90") { + t.Errorf("the run does not say it ignores #90:\n%s", stderr) + } + }) + } +} + +// The bot comes from BACKPORT_BATCH_BOT when it is set, empty included, and +// is the scheduled run's own bot when it is not. +func TestBatchBotLogin(t *testing.T) { + env := func(v string, ok bool) func(string) (string, bool) { + return func(key string) (string, bool) { + if key != batchBotEnv { + t.Errorf("looked up %s", key) + } + return v, ok + } + } + for _, tc := range []struct { + name, v string + ok bool + want string + }{ + {"unset", "", false, "cozystack-ci[bot]"}, + {"set empty", "", true, ""}, + {"set", "other-app[bot]", true, "other-app[bot]"}, + } { + if got := batchBotLogin(env(tc.v, tc.ok)); got != tc.want { + t.Errorf("%s: batchBotLogin = %q, want %q", tc.name, got, tc.want) + } + } +} diff --git a/cmd/backport-batch/body.go b/cmd/backport-batch/body.go new file mode 100644 index 000000000..da4691450 --- /dev/null +++ b/cmd/backport-batch/body.go @@ -0,0 +1,514 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode" +) + +// Titles, release notes and file names in the body were written by whoever +// opened the originals, and the body is published under the name of whoever +// runs --push. None of it is searched for what might be dangerous and then +// rendered by other rules: a title, a reason or a file name goes whole into a +// code span, and the release notes into the fenced release-note block, both +// of which GitHub shows as written, with no mention, reference, link or HTML +// in them acted on. neutralNote is for a note copied out of that block. + +const ( + ourRepoURL = "https://github.com/" + upstreamRepo + "/" + // tasksURL is where the instructions for the work a batch leaves live. + tasksURL = ourRepoURL + "blob/main/docs/release/backport-tasks/" +) + +const ( + titleLimit = 200 + // GitHub refuses a pull request body longer than this. + bodyLimit = 65536 +) + +var ( + backtickRunRE = regexp.MustCompile("`+") + fenceRunRE = regexp.MustCompile("`{3,}|~{3,}") + mdLinkRE = regexp.MustCompile(`!?\[([^\]]*)\]\(([^)]*)\)`) + urlRE = regexp.MustCompile("(?i)\\b(?:https?|ftp)://[^\\s<>()\\[\\]`]+|\\bwww\\.[^\\s<>()\\[\\]`]+") + // A closing keyword with its target, a cross-repository reference and a + // mention, wherever they stand: no exception for what precedes them, so + // nothing depends on agreeing with GitHub about word boundaries. + actingRE = regexp.MustCompile(`(?i)\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\b\s*:?\s*(?:(?:[\w.-]+/[\w.-]+)?#\d+|https?://github\.com/[\w.-]+/[\w.-]+/(?:issues|pull)/\d+)` + + `|[\w.-]+/[\w.-]+#\d+|@[A-Za-z0-9][A-Za-z0-9-]*(?:/[A-Za-z0-9_.-]+)?`) + // A link into this repository survives, unless a dot segment or an escape + // could take it somewhere else. + safeURLRE = regexp.MustCompile(`^` + regexp.QuoteMeta(ourRepoURL) + `[\w/?=&#+.-]*$`) + releaseNotRE = regexp.MustCompile("(?ms)^[ \t]*```+[ \t]*release-note[ \t]*\n(.*?)^[ \t]*```") + headingRE = regexp.MustCompile(`^#{1,6} `) + placeholders = []string{"", "none", "n/a", "na", "no", "-", "nothing"} +) + +// clean drops what is not text: control characters other than tab and, when +// asked, newline, which a terminal acts on, and format characters such as the +// zero-width and bidi ones, which make text read as something it is not. +// Invalid UTF-8, a raw 8-bit escape among it, becomes U+FFFD. +func clean(s string, newlines bool) string { + var b strings.Builder + for _, r := range strings.ToValidUTF8(s, "\uFFFD") { + switch { + case r == '\t', r == '\n' && newlines: + b.WriteRune(r) + case r == '\r' || unicode.Is(unicode.Cf, r): + case r == '\n', r == '\u2028', r == '\u2029': + b.WriteRune(' ') + case unicode.IsControl(r): + b.WriteRune('\uFFFD') + default: + b.WriteRune(r) + } + } + return b.String() +} + +func oneLine(s string) string { + return strings.Join(strings.FieldsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }), " ") +} + +func capped(s string, limit int) string { + if r := []rune(s); len(r) > limit { + return strings.TrimRightFunc(string(r[:limit-1]), unicode.IsSpace) + "…" + } + return s +} + +// codeSpan puts s on one line in a code span: its fence is longer than any +// backtick run inside, so nothing in s can close it. +func codeSpan(s string) string { + s = oneLine(clean(s, false)) + longest := 0 + for _, run := range backtickRunRE.FindAllString(s, -1) { + longest = max(longest, len(run)) + } + fence := strings.Repeat("`", longest+1) + pad := "" + if s == "" || strings.HasPrefix(s, "`") || strings.HasSuffix(s, "`") { + pad = " " + } + return fence + pad + s + pad + fence +} + +// titleSpan is a carried title as the body shows it. +func titleSpan(s string) string { return codeSpan(capped(oneLine(clean(s, false)), titleLimit)) } + +func prLink(n int) string { + return fmt.Sprintf("[#%d](%spull/%d)", n, ourRepoURL, n) +} + +func ours(url string) bool { + return safeURLRE.MatchString(url) && !strings.Contains(url, "/.") +} + +// inertSigns swaps the characters GitHub acts on for their fullwidth forms, +// which read the same and act on nothing. Unlike a code span, the swap does +// not depend on how the backticks around it pair up. +var inertSigns = strings.NewReplacer("@", "@", "#", "#", "://", "://") + +// neutralNote makes a release note safe inside the fenced block and inert +// outside it: no run of backticks or tildes is left to close the fence, links +// lose their target unless it is this repository, and closing keywords, +// cross-repository references and mentions lose the signs that make them +// act. Each step runs on the output of the one before, so a mention a link's +// text carried is still found. +func neutralNote(s string) string { + s = clean(s, true) + s = mdLinkRE.ReplaceAllString(s, "$1 ($2)") + s = urlRE.ReplaceAllStringFunc(s, func(u string) string { + if ours(u) { + return u + } + return "(link removed)" + }) + s = actingRE.ReplaceAllStringFunc(s, inertSigns.Replace) + return fenceRunRE.ReplaceAllStringFunc(s, func(f string) string { return strings.Repeat("'", len(f)) }) +} + +// releaseNote is the release-note block of a body, or "" when it has none or +// only a placeholder. +func releaseNote(body string) string { + m := releaseNotRE.FindStringSubmatch(htmlCommentRE.ReplaceAllString(body, "")) + if m == nil { + return "" + } + note := strings.TrimSpace(m[1]) + if slices.Contains(placeholders, strings.Trim(strings.ToLower(note), " .")) { + return "" + } + return note +} + +// deferredFence opens the block check reads the deferred conflicts from. +const deferredFence = "```backport-batch" + +// deferredEntry is a conflict, with the commit that failed and its files, an +// original --defer took out of the batch, with the reason given, or one that +// waits for a maintainer because it changes workflows, with those files. +type deferredEntry struct { + Original int `json:"original"` + Commit string `json:"commit,omitempty"` + Files []string `json:"files"` + More int `json:"more,omitempty"` + Reason string `json:"reason,omitempty"` + Workflows bool `json:"workflows,omitempty"` +} + +// How many conflicting files of one original the body names, in the list and +// in the block: a vendored chart can conflict in hundreds. +const ( + filesListed = 5 + filesKept = 50 +) + +type deferredRecord struct { + Line string `json:"line"` + Deferred []deferredEntry `json:"deferred"` +} + +// deferredBlock is a version 1 record, which batches carried before records +// had a version. encoding/json leaves bidi and zero-width characters as they +// are, so file names go through clean first, as the reason does. +func deferredBlock(l line, conflicts, held, waiting []pick) string { + data, _ := json.Marshal(deferredRecord{Line: l.branch(), Deferred: deferredEntries(conflicts, held, waiting)}) + return deferredFence + "\n" + string(data) + "\n```" +} + +func cleanFiles(files []string) []string { + out := make([]string, len(files)) + for i, f := range files { + out[i] = clean(f, false) + } + return out +} + +// parseDeferred reads what a batch body's record defers. +func parseDeferred(body string, l line) (map[int]deferredEntry, error) { + rec, err := parseRecord(body, l) + if err != nil { + return nil, err + } + return rec.deferred(), nil +} + +// prTitle is built from the line and the date alone, never from carried text. +func prTitle(l line, date string) string { + return fmt.Sprintf("fix(backport): batch for %s, %s", l.branch(), date) +} + +// skipRules are the rules a skipped original is reported under, in order. +var skipRules = []int{ruleBeforeStart, ruleOnLine, ruleType, ruleBreaking, ruleTouches, ruleNewer, ruleDropped} + +// report is what one run put on the batch branch and what it left out. +type report struct { + line line + newer *line + onBranch []*verdict // carried from --onto and picked now, in merge order + deferred []pick // conflicts + held []pick // what --defer took out, with the reason + waiting []pick // what changes workflows, left to a maintainer + skipped map[int][]*verdict + other int + // overrides are the originals a backport/ label kept out or let in, and + // stale those with backport/skip that are still on the batch. + overrides []*verdict + stale []*verdict + start time.Time + // shapes says, for an original on the branch, what it brought. + shapes map[*verdict]string + // missing are originals the policy takes that the run neither tried nor + // found anywhere. + missing []*verdict + // opts is how the run was asked to classify, and built what it read. + opts runOptions + built builtFrom + // blocks are the record and the report the body ends with. + blocks string + // closing is the close of the open batch this run leaves carrying + // nothing. + closing *closing + // dropped are the maintainers' drops the record keeps. + dropped []droppedEntry +} + +// ruleName is how output names what kept an original out: the policy's +// number for its own rules, and words for what the policy does not number. +func ruleName(rule int) string { + switch rule { + case ruleSkipLabel: + return labelSkip + case ruleBeforeStart: + return "rule 1, before the line's start" + case ruleDropped: + return "a maintainer's drop" + } + return "rule " + strconv.Itoa(rule) +} + +// listTitle titles the list of what a rule skipped, a drop among them. +func (r *report) listTitle(rule int) string { + if rule == ruleDropped { + return "Dropped from " + codeSpan(r.line.branch()) + " by a maintainer" + } + return r.ruleTitle(rule) +} + +func (r *report) ruleTitle(rule int) string { + switch rule { + case ruleBeforeStart: + return "Rule 1, merged before " + codeSpan(r.line.branch()) + " started under the policy on " + r.start.Format(time.DateOnly) + case ruleOnLine: + return "Rule 1, already on " + codeSpan(r.line.branch()) + case ruleType: + return "Rule 2, the title type is not `fix` or `security`, or the scope is `deps`" + case ruleBreaking: + return "Rule 3, a breaking change" + case ruleTouches: + return "Rule 4, touches platform migrations, `targetVersion` or a CRD, or makes a sizeable API change" + case ruleNewer: + return "Rule 6, not on " + codeSpan(r.newer.branch()) + " yet" + } + return fmt.Sprintf("Rule %d", rule) +} + +// what is the What this PR does section, with the skipped lists of the rules +// in details written out. +func (r *report) what(details map[int]bool) string { + var b strings.Builder + fmt.Fprintf(&b, "Backports to %s from `main`, picked by the rules of the [backport policy](%s) in merge order. A merged pull request comes as its own commits, picked with `git cherry-pick -x` so each keeps its author, message and sign-offs, and a merge commit that carries the original's merge message with a `%s:` trailer naming it; a squash-merged one comes as one pick. `backport-batch verify` checks that shape. A pick listed as changed differs from the commit it came from; `git range-diff ^1..^2 ^1..^2` shows how. A conflict or a failing check here is a [backport task](%sREADME.md).\n\n", codeSpan(r.line.branch()), policyURL, batchTrailer, tasksURL) + if len(r.onBranch) == 0 { + b.WriteString("Nothing is picked yet.\n") + } + for _, v := range r.onBranch { + shape := "" + if s := r.shapes[v]; s != "" { + shape = ", " + s + } + fmt.Fprintf(&b, "- %s %s%s\n", prLink(v.o.number), titleSpan(v.o.title), shape) + } + if len(r.deferred) > 0 { + fmt.Fprintf(&b, "\n### Deferred conflicts\n\nThese do not pick cleanly onto this branch. Each names the first of its commits that conflicted and the files it conflicted in, and nothing of it is on the branch. Each moves to the next batch, unless a resolution checked with `backport-batch fold` is pushed here first, as [resolving a conflict](%sresolve-conflict.md) describes.\n\n", tasksURL) + for _, p := range r.deferred { + var files []string + for _, f := range p.files[:min(len(p.files), filesListed)] { + files = append(files, codeSpan(f)) + } + if more := len(p.files) - len(files) + p.more; more > 0 { + files = append(files, fmt.Sprintf("%d more", more)) + } + at := "" + if p.failed != "" { + at = " at " + codeSpan(p.failed[:12]) + } + fmt.Fprintf(&b, "- %s %s%s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), at, strings.Join(files, ", ")) + } + } + if len(r.held) > 0 { + b.WriteString("\n### Deferred to the next batch\n\nTaken out of this batch with `backport-batch run --defer`, each for the reason given. This pull request keeps them out; the next batch picks them again.\n\n") + for _, p := range r.held { + fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), titleSpan(p.reason)) + } + } + if len(r.waiting) > 0 { + fmt.Fprintf(&b, "\n### Waiting for a maintainer\n\nThese change `%s`, which the token this batch runs with may not push, so it leaves them out. The policy still takes each, and `backport-batch check` lists it until it is on the line. A maintainer brings one over as [changes to workflows](%s#changes-to-workflows) describes.\n\n", workflowsDir, policyURL) + for _, p := range r.waiting { + var files []string + for _, f := range p.files[:min(len(p.files), filesListed)] { + files = append(files, codeSpan(f)) + } + if more := len(p.files) - len(files) + p.more; more > 0 { + files = append(files, fmt.Sprintf("%d more", more)) + } + fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), strings.Join(files, ", ")) + } + } + if len(r.overrides) > 0 { + b.WriteString("\n### Maintainer overrides\n\n") + for _, v := range r.overrides { + fmt.Fprintf(&b, "- %s %s: %s, %s\n", prLink(v.o.number), titleSpan(v.o.title), codeSpan(v.label), r.outcome(v)) + } + } + b.WriteString("\n### Skipped\n\n") + for _, rule := range skipRules { + items := r.skipped[rule] + if len(items) == 0 { + continue + } + left := "" + if !details[rule] { + left = ", too many to list here; the run's output lists them" + } + fmt.Fprintf(&b, "- %s: %d%s\n", r.listTitle(rule), len(items), left) + } + if r.other > 0 { + fmt.Fprintf(&b, "- Commits on `main` that merged no pull request: %d\n", r.other) + } + for _, rule := range skipRules { + items := r.skipped[rule] + if len(items) > 0 && details[rule] { + fmt.Fprintf(&b, "\n
%s\n\n", r.listTitle(rule)) + for _, v := range items { + fmt.Fprintf(&b, "- %s %s, %s\n", prLink(v.o.number), titleSpan(v.o.title), titleSpan(v.reason)) + } + b.WriteString("\n
\n") + } + } + b.WriteString("\n" + r.blocks + "\n") + return b.String() +} + +// outcome is what became of an original a label overrode. +func (r *report) outcome(v *verdict) string { + switch { + case slices.Contains(r.stale, v) && v.rule == ruleDropped: + return "dropped, but still on this branch; take it out with `backport-batch run --defer`" + case slices.Contains(r.stale, v): + return "still on this branch, picked before the label; take it out with `backport-batch run --defer`" + case v.label == labelSkip: + return "left out" + case slices.Contains(r.onBranch, v): + return "on this branch" + case slices.ContainsFunc(r.deferred, func(p pick) bool { return p.v == v }): + return "a deferred conflict" + case slices.ContainsFunc(r.held, func(p pick) bool { return p.v == v }): + return "deferred to the next batch" + case slices.ContainsFunc(r.waiting, func(p pick) bool { return p.v == v }): + return "waiting for a maintainer: it changes " + codeSpan(workflowsDir) + case v.rule != 0: + return "skipped by " + ruleName(v.rule) + ", " + titleSpan(v.reason) + } + return "not picked" +} + +func (r *report) notes() string { + var notes []string + for _, v := range r.onBranch { + if n := releaseNote(v.o.body); n != "" { + notes = append(notes, neutralNote(n)) + } + } + return strings.Join(notes, "\n\n") +} + +// section is one heading of the pull request template and what follows it. +type section struct { + heading string + text string +} + +func sections(template string) (string, []section) { + var lead strings.Builder + var out []section + inFence := false + for _, ln := range strings.Split(htmlCommentRE.ReplaceAllString(template, ""), "\n") { + if strings.HasPrefix(strings.TrimSpace(ln), "```") { + inFence = !inFence + } + if !inFence && headingRE.MatchString(ln) { + out = append(out, section{heading: ln}) + continue + } + if len(out) == 0 { + lead.WriteString(ln + "\n") + } else { + out[len(out)-1].text += ln + "\n" + } + } + return strings.TrimSpace(lead.String()), out +} + +func headingIs(heading, name string) bool { + return strings.EqualFold(strings.TrimSpace(strings.TrimLeft(heading, "#")), name) +} + +// body fills the line's own pull request template: the What, Screenshots and +// Release note sections are the batch's, every other section stays as the +// template has it. A body over GitHub's limit loses its longest skipped lists +// first, and its release notes' tail last; left out are the rules whose lists +// it lost. +func (r *report) body(template string) (body string, left []int) { + details := map[int]bool{} + for rule, items := range r.skipped { + details[rule] = len(items) > 0 + } + notes := r.notes() + for { + full := r.render(template, r.what(details), notes) + if len(full) <= bodyLimit { + return full, left + } + longest := 0 + for _, rule := range skipRules { + if details[rule] && (longest == 0 || len(r.skipped[rule]) > len(r.skipped[longest])) { + longest = rule + } + } + if longest == 0 { + const marker = "\n\n(release notes truncated; see the original pull requests)" + cut := max(0, len(notes)-(len(full)-bodyLimit)-len(marker)) + for cut > 0 && !utf8Start(notes[cut]) { + cut-- + } + return r.render(template, r.what(details), notes[:cut]+marker), left + } + details[longest] = false + left = append(left, longest) + } +} + +func utf8Start(c byte) bool { return c&0xC0 != 0x80 } + +func (r *report) render(template, what, notes string) string { + lead, secs := sections(template) + var b strings.Builder + if lead != "" { + b.WriteString(lead + "\n\n") + } + seen := map[string]bool{} + for _, s := range secs { + switch { + case headingIs(s.heading, "What this PR does"): + seen["what"] = true + b.WriteString(s.heading + "\n\n" + what + "\n") + case headingIs(s.heading, "Screenshots"): + b.WriteString(s.heading + "\n\nSee the original pull requests.\n\n") + case headingIs(s.heading, "Release note"): + seen["note"] = true + b.WriteString(s.heading + "\n\n```release-note\n" + notes + "\n```\n\n") + default: + b.WriteString(s.heading + "\n\n" + strings.TrimSpace(s.text) + "\n\n") + } + } + out := b.String() + if !seen["what"] { + out = "## What this PR does\n\n" + what + "\n" + out + } + if !seen["note"] { + out += "### Release note\n\n```release-note\n" + notes + "\n```\n" + } + return strings.TrimSpace(regexp.MustCompile(`\n{3,}`).ReplaceAllString(out, "\n\n")) + "\n" +} diff --git a/cmd/backport-batch/body_test.go b/cmd/backport-batch/body_test.go new file mode 100644 index 000000000..3f746d17d --- /dev/null +++ b/cmd/backport-batch/body_test.go @@ -0,0 +1,382 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + "unicode" +) + +// visible is what of a body GitHub renders as markdown rather than shows as +// code: fenced blocks and code spans taken out, delimited as CommonMark +// delimits them. Everything carried has to be gone from it. +func visible(body string) string { + fenceOpen := regexp.MustCompile("^(`{3,}|~{3,})") + var out []string + fence := "" + for _, ln := range strings.Split(body, "\n") { + t := strings.TrimLeft(ln, " ") + indented := len(ln)-len(t) > 3 + if fence != "" { + run := strings.TrimLeft(t, fence[:1]) + if !indented && len(t)-len(run) >= len(fence) && strings.TrimSpace(run) == "" { + fence = "" + } + continue + } + if m := fenceOpen.FindString(t); m != "" && !indented && (m[0] == '~' || !strings.Contains(t[len(m):], "`")) { + fence = m + continue + } + out = append(out, stripSpans(ln)) + } + return strings.Join(out, "\n") +} + +func stripSpans(s string) string { + var b strings.Builder + for i := 0; i < len(s); { + switch { + case s[i] == '\\' && i+1 < len(s): + b.WriteString(s[i : i+2]) + i += 2 + continue + case s[i] != '`': + b.WriteByte(s[i]) + i++ + continue + } + j := i + for j < len(s) && s[j] == '`' { + j++ + } + n, closing := j-i, -1 + for k := j; k < len(s); { + if s[k] != '`' { + k++ + continue + } + e := k + for e < len(s) && s[e] == '`' { + e++ + } + if e-k == n { + closing = k + break + } + k = e + } + if closing < 0 { + b.WriteString(s[i:j]) + i = j + continue + } + i = closing + n + } + return b.String() +} + +// hostile is carried text built to get out of its span or block: each carries +// a marker that must never be visible. +var hostile = []string{ + "zqx01 thanks @alice", + "zqx02 (@org/team) please look", + "zqx03 [@bob](https://evil.example/x) [docs](https://github.com/cozystack/cozystack/../../evil/repo)", + "zqx04 zero\u200b@carol bidi \u202e@dave \ufeff@erin", + "zqx05 ``` `@frank` ``` still inside", + "zqx06 ` closes a single span, then @grace", + "zqx07 `` `` ``` ```` four", + "zqx08 Fixes #1, closes other/repo#2, resolves https://github.com/other/repo/issues/3", + "zqx09 x", + "zqx10 www.evil.example ", + "zqx11 [ref]: https://evil.example", + "zqx12 @heidi & <b>", + "zqx13 \x1b[31mred\x1b[0m \u009b2J \x07 bell", + "zqx14
x", + "`zqx15` starts and ends with a backtick`", +} + +// hostileNote is a release note whose own block a bare fence line would end, +// so it hides one behind a zero-width space, which clean then removes. +func hostileNote(i int, s string) string { + return fmt.Sprintf("fix(x): zqn%02d %s\n\u200b```\n@ivan zqn%02d after a fence\n~~~\n[@judy](https://evil.example) Fixes #9 \x1b[2J", i, s, i) +} + +// Every carried title, reason and release note stays inside a code span or the +// release-note block, however it is written. +func TestBodyContainsCarriedText(t *testing.T) { + f := newFixture(t) + f.lines() + for i, s := range hostile { + typ := "fix" + if i%2 == 1 { + typ = "feat" // skipped, so it lands in a
list + } + body := "## What this PR does\n\n```release-note\n" + hostileNote(i, s) + "\n```\n" + f.pr(100+i, typ+"(x): "+s, body, map[string]string{fmt.Sprintf("app/%d.txt", i): "x\n"}) + } + gh := &fakeGH{} + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if len(gh.bodies) != 1 { + t.Fatalf("gh got %d bodies, want 1", len(gh.bodies)) + } + for _, body := range []string{gh.bodies[0], stdout} { + for i := range hostile { + title, note := fmt.Sprintf("zqx%02d", i+1), fmt.Sprintf("zqn%02d", i) + if i == len(hostile)-1 { + title = "zqx15" + } + if !strings.Contains(body, title) { + t.Errorf("%s is missing from the body", title) + } + if i%2 == 0 && !strings.Contains(body, note) { + t.Errorf("%s is missing from the release notes", note) + } + } + if v := visible(body); strings.Contains(v, "zqx") || strings.Contains(v, "zqn") { + t.Errorf("carried text is visible outside code:\n%s", v) + } + for _, r := range body { + if r != '\n' && r != '\t' && unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { + t.Errorf("the body carries %U", r) + break + } + } + } +} + +// Copied out of the fenced block, a note must still close, ping and link +// nothing. +func TestNeutralNote(t *testing.T) { + cases := []struct{ in, gone string }{ + {"thanks @alice", "@alice"}, + {"(@org/team) look", "@org/team"}, + {"[@bob](https://evil.example)", "@bob"}, + {"x\u200b@carol", "@carol"}, + {"a@dave.example", "@dave"}, + {"Fixes #1", "Fixes #1"}, + {"closes other/repo#2", "other/repo#2"}, + {"see other/repo#3", "other/repo#3"}, + {"resolves https://github.com/other/repo/issues/3", "github.com/other"}, + {"Fixes https://github.com/cozystack/cozystack/issues/3", "Fixes https://"}, + {"x``@erin", "@erin"}, + {"`code @frank here`", "@frank"}, + {"```@grace```", "@grace"}, + {"go to https://evil.example/x", "evil.example"}, + {"go to www.evil.example", "evil.example"}, + {"[docs](https://evil.example)", "evil.example"}, + {"[docs](https://github.com/cozystack/cozystack/../../evil/repo)", "evil/repo"}, + {"line\n```\nescaped", "```"}, + {"line\n~~~\nescaped", "~~~"}, + {"zero\u200b``\u200b`", "```"}, + {"\x1b[2J", "\x1b"}, + } + for _, tc := range cases { + got := neutralNote(tc.in) + if strings.Contains(visible(got), tc.gone) { + t.Errorf("neutralNote(%q) = %q, still shows %q", tc.in, got, tc.gone) + } + } + keep := "see https://github.com/cozystack/cozystack/blob/main/docs/release/backports.md" + if got := neutralNote(keep); got != keep { + t.Errorf("neutralNote(%q) = %q, want a link into this repository kept", keep, got) + } +} + +func TestCodeSpan(t *testing.T) { + cases := []string{"plain", "a `b` c", "``x``", "`lead", "trail`", "", "a\nb", "````", "\x1b[31m`\u202e`"} + for _, s := range cases { + got := codeSpan(s) + if v := visible(got); v != "" { + t.Errorf("codeSpan(%q) = %q leaves %q outside the span", s, got, v) + } + if strings.ContainsAny(got, "\n\x1b\u202e") { + t.Errorf("codeSpan(%q) = %q carries a newline or a control", s, got) + } + } +} + +func TestDeferredRoundTrip(t *testing.T) { + l, _ := parseLine("release-1.6") + v := &verdict{o: &original{number: 3735}} + held := &verdict{o: &original{number: 3736}} + block := deferredBlock(l, []pick{{v: v, files: []string{"a\n```\nb", "&d", "e`f"}}}, []pick{{v: held, reason: "breaks\n```\ne2e"}}, nil) + if lines := strings.Split(block, "\n"); len(lines) != 3 { + t.Fatalf("the block is %d lines, want 3:\n%s", len(lines), block) + } + body := "## What this PR does\n\ntext\n\n" + block + "\n\n### Release note\n" + got, err := parseDeferred(body, l) + if err != nil || len(got) != 2 || len(got[3735].Files) != 3 || got[3735].Reason != "" { + t.Errorf("parseDeferred = %v, %v; want the conflict #3735 with its files", got, err) + } + if e := got[3736]; e.Reason != "breaks ``` e2e" || len(e.Files) != 0 { + t.Errorf("parseDeferred = %+v; want #3736 held, for its reason on one line", e) + } + if strings.ContainsAny(strings.Split(block, "\n")[1], "<>&") { + t.Errorf("the JSON line carries raw HTML characters: %s", block) + } + + other, _ := parseLine("release-1.5") + for name, body := range map[string]string{ + "another line": body, + "two blocks": body + "\n" + block, + "not JSON": deferredFence + "\nnot json\n```", + "an unknown key": deferredFence + "\n{\"line\":\"release-1.6\",\"deferred\":[],\"x\":1}\n```", + "a bad number": deferredFence + "\n{\"line\":\"release-1.6\",\"deferred\":[{\"original\":-1}]}\n```", + "a repeat": deferredFence + "\n{\"line\":\"release-1.6\",\"deferred\":[{\"original\":5},{\"original\":5,\"reason\":\"x\"}]}\n```", + "a bad commit": deferredFence + "\n{\"line\":\"release-1.6\",\"deferred\":[{\"original\":5,\"commit\":\"-x\"}]}\n```", + "a broken fence": deferredFence + "\n{}\nmore\n```", + "an empty fence": deferredFence + "\n```", + "a trailing line": deferredFence, + } { + want := l + if name == "another line" { + want = other + } + if _, err := parseDeferred(body, want); err == nil { + t.Errorf("%s: parseDeferred accepted it", name) + } + } + if got, err := parseDeferred("no block here", l); err != nil || len(got) != 0 { + t.Errorf("no block: %v, %v; want nothing deferred", got, err) + } +} + +func TestReleaseNote(t *testing.T) { + cases := map[string]string{ + "### Release note\n\n```release-note\nfix(x): y\n```\n": "fix(x): y", + "```release-note\nNONE\n```": "", + "```release-note\n\n```": "", + "": "", + "no block": "", + } + for body, want := range cases { + if got := releaseNote(body); got != want { + t.Errorf("releaseNote(%q) = %q, want %q", body, got, want) + } + } +} + +// The body is the line's template with the batch's sections filled in, and +// fits the size GitHub accepts. +func TestBodyFollowsTheTemplate(t *testing.T) { + l, _ := parseLine("release-1.6") + template := "\n\n## What this PR does\n\n\n### Screenshots\n\n\n\n### Downstream repositories\n\n- [ ] No downstream repository is affected by this change\n\n### Release note\n\n```release-note\n\n```\n" + o := &original{number: 7, title: "fix(x): y", body: "```release-note\nfix(x): note\n```"} + r := &report{line: l, onBranch: []*verdict{{o: o}}, skipped: map[int][]*verdict{}, blocks: deferredBlock(l, nil, nil, nil)} + body, _ := r.body(template) + for _, want := range []string{"## What this PR does\n\nBackports to", "- [#7](https://github.com/cozystack/cozystack/pull/7) `fix(x): y`", "### Downstream repositories\n\n- [ ] No downstream", "```release-note\nfix(x): note\n```", deferredFence} { + if !strings.Contains(body, want) { + t.Errorf("the body lacks %q:\n%s", want, body) + } + } + if strings.Contains(body, "" + +// stalledAfter is how many days a batch stays open before notify calls it +// stalled: the shorter gap between two scheduled runs, Monday to Thursday, so +// a batch the previous run opened or continued is stalled at the next one. +const stalledAfter = 3 + +var mentionRE = regexp.MustCompile(`^@?([A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:/[A-Za-z0-9_.-]+)?)$`) + +// notify reports what waits on the open batch pull request of a line for a +// maintainer: a batch open since before the previous run, the conflicts it +// deferred, and the originals that change workflows. One line per finding goes +// to stdout, for the run's summary. --push writes them into the comment, one +// per pull request, updated in place on every run. +func (a *app) notify(args []string) int { + fs := a.flags("notify") + lineArg := fs.String("line", "", "release line, release-X.Y") + mention := fs.String("mention", "", "team or person the comment mentions, org/team or login, with or without @") + push := fs.Bool("push", false, "write the comment") + if err := parse(fs, args); err != nil { + return 2 + } + l, err := parseLine(*lineArg) + if err != nil { + return a.fail(err) + } + if *mention != "" { + m := mentionRE.FindStringSubmatch(*mention) + if m == nil { + return a.fail(fmt.Errorf("--mention %q is not @login or @org/team", *mention)) + } + *mention = "@" + m[1] + } + prs, err := a.gh.openPulls(l.branch()) + if err != nil { + return a.fail(err) + } + pr, warnings, err := batchPull(a.gh, prs, l) + for _, w := range warnings { + a.warn("%s", w) + } + if err != nil { + return a.fail(err) + } + // With no batch open there is nothing to report on and nowhere to write, + // but a record nobody can read is said all the same. + src, err := findLineRecord(a.gh, l, pr) + if err != nil { + return a.fail(err) + } + if c := src.closingOpen(); c != nil { + return a.fail(fmt.Errorf("batch pull request #%d is being closed; backport-batch run --line %s --push finishes it", c.PR, l.branch())) + } + if pr == nil { + return 0 + } + opened, err := branchDate(pr.Head.Ref) + if err != nil { + return a.fail(err) + } + deferred := map[int]deferredEntry{} + if src.open { + deferred = src.rec.deferred() + } + var conflicts, waiting []int + for n, e := range deferred { + switch { + case e.Workflows: + waiting = append(waiting, n) + case e.Reason == "": + conflicts = append(conflicts, n) + } + } + slices.Sort(conflicts) + slices.Sort(waiting) + + today := a.now().UTC().Truncate(24 * time.Hour) + days := int(today.Sub(opened).Hours() / 24) + stalled := days >= stalledAfter + var text []string + if stalled { + fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) + text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days)) + } + if len(conflicts) > 0 { + fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), len(conflicts), pr.Number) + var b strings.Builder + fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch:\n", tasksURL) + for _, n := range conflicts { + fmt.Fprintf(&b, "\n- %s", prLink(n)) + } + text = append(text, b.String()) + } + if len(waiting) > 0 { + fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + var b strings.Builder + fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes:\n", workflowsDir, policyURL) + for _, n := range waiting { + fmt.Fprintf(&b, "\n- %s", prLink(n)) + } + text = append(text, b.String()) + } + if !*push { + return 0 + } + return a.comment(pr, *mention, today, text) +} + +// comment writes what waits into the one notify comment on the batch pull +// request: it edits the comment if there is one, and opens it only when +// something waits. Only a comment by an author the batch trusts is taken for +// it, so a look-alike someone else wrote is neither edited nor believed. +func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []string) int { + comments, err := a.gh.comments(pr.Number) + if err != nil { + return a.fail(err) + } + var ours *issueComment + for i := range comments { + c := &comments[i] + if !strings.HasPrefix(c.Body, notifyMarker) { + continue + } + why, err := trustedAuthor(a.gh, c.User.Login, c.User.Type) + if err != nil { + return a.fail(err) + } + if why == "" { + ours = c + } + } + if ours == nil && len(text) == 0 { + return 0 + } + var b strings.Builder + b.WriteString(notifyMarker + "\n") + if mention != "" && len(text) > 0 { + b.WriteString(mention + "\n\n") + } + if len(text) == 0 { + text = []string{"Nothing waits on a maintainer."} + } + b.WriteString(strings.Join(text, "\n\n")) + fmt.Fprintf(&b, "\n\nUpdated by the backport batch on %s.\n", today.Format(time.DateOnly)) + f, err := os.CreateTemp("", "backport-batch-comment-*.md") + if err != nil { + return a.fail(err) + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(b.String()); err != nil { + f.Close() + return a.fail(err) + } + if err := f.Close(); err != nil { + return a.fail(err) + } + if err := a.stillOpen(pr.Number); err != nil { + return a.fail(err) + } + if ours != nil { + err = a.gh.editComment(ours.ID, f.Name()) + } else { + err = a.gh.createComment(pr.Number, f.Name()) + } + if err != nil { + return a.fail(err) + } + return 0 +} + +// lines prints the lines the batch runs for, newest first, in the order the +// policy has them processed. +func (a *app) lines(args []string) int { + fs := a.policyFlags("lines") + if err := parse(fs, args); err != nil { + return 2 + } + if err := fetch(); err != nil { + return a.fail(err) + } + mainHead, err := resolve("refs/remotes/" + remote + "/main") + if err != nil { + return a.fail(err) + } + pol, _, err := a.policy(mainHead) + if err != nil { + return a.fail(err) + } + lines, err := batchLines(pol.released) + if err != nil { + return a.fail(err) + } + for _, l := range slices.Backward(lines) { + fmt.Fprintln(a.stdout, l.branch()) + } + return 0 +} diff --git a/cmd/backport-batch/notify_test.go b/cmd/backport-batch/notify_test.go new file mode 100644 index 000000000..4bfb650e9 --- /dev/null +++ b/cmd/backport-batch/notify_test.go @@ -0,0 +1,336 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "slices" + "strings" + "testing" + "time" +) + +// notifyPR is an open batch pull request for release-1.1 on the branch opened +// on date, YYYYMMDD, with body. +func notifyPR(date, body string) pullRequest { + var pr pullRequest + pr.Number, pr.Body = 90, body + pr.User.Login, pr.User.Type = "maintainer", "User" + pr.Head.Ref, pr.Head.SHA = "backport-batch/release-1.1-"+date, strings.Repeat("a", 40) + pr.Head.Repo = &struct { + FullName string `json:"full_name"` + }{upstreamRepo} + pr.Base.Ref, pr.Base.Repo.FullName = "release-1.1", upstreamRepo + return pr +} + +// notifyRun runs notify on 2026-10-26 with gh faked. +func notifyRun(gh *fakeGH, args ...string) (int, string, string) { + var stdout, stderr bytes.Buffer + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }} + code := a.main(append([]string{"notify", "--line", "release-1.1"}, args...)) + return code, stdout.String(), stderr.String() +} + +var notifyLine, _ = parseLine("release-1.1") + +// A batch open since before the previous run gets one comment, mentioning +// the team, and keeps that one comment however many runs see it. +func TestNotifyStalled(t *testing.T) { + for _, tc := range []struct { + date string + stalled bool + }{ + {"20261026", false}, + {"20261024", false}, + {"20261023", true}, + {"20261019", true}, + } { + gh := &fakeGH{prs: []pullRequest{notifyPR(tc.date, "")}} + code, stdout, stderr := notifyRun(gh, "--mention", "cozystack/maintainers", "--push") + if code != 0 { + t.Fatalf("%s: notify = %d\n%s", tc.date, code, stderr) + } + if got := strings.Contains(stdout, "release-1.1: batch pull request #90 has been open since"); got != tc.stalled { + t.Errorf("%s: stalled = %v, want %v:\n%s", tc.date, got, tc.stalled, stdout) + } + if got := len(gh.issueComments); got != map[bool]int{true: 1, false: 0}[tc.stalled] { + t.Errorf("%s: %d comments, want one only when stalled", tc.date, got) + } + if !tc.stalled { + continue + } + c := gh.issueComments[0].Body + if !strings.HasPrefix(c, notifyMarker+"\n@cozystack/maintainers\n") || !strings.Contains(c, "open since") { + t.Errorf("%s: the comment does not mention the team and say why:\n%s", tc.date, c) + } + for range 2 { + if code, _, stderr := notifyRun(gh, "--mention", "@cozystack/maintainers", "--push"); code != 0 { + t.Fatalf("notify again = %d\n%s", code, stderr) + } + } + if len(gh.issueComments) != 1 { + t.Errorf("%s: %d comments after three runs, want the one updated", tc.date, len(gh.issueComments)) + } + } + + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}} + if code, _, _ := notifyRun(gh, "--push"); code != 0 || strings.Contains(gh.issueComments[0].Body, "@") { + t.Errorf("notify with no team = %d and wrote %q; want no mention", code, gh.issueComments[0].Body) + } + gh = &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}} + if code, stdout, _ := notifyRun(gh); code != 0 || stdout == "" || len(gh.issueComments) != 0 { + t.Errorf("notify without --push = %d, %q and %d comments; want the line and no comment", code, stdout, len(gh.issueComments)) + } +} + +// The comment lists the deferred conflicts for a maintainer, and nothing +// --defer held out. +func TestNotifyConflicts(t *testing.T) { + conflict := pick{v: &verdict{o: &original{number: 6}}, files: []string{"app/six.txt"}} + held := pick{v: &verdict{o: &original{number: 7}}, reason: "fails the e2e"} + body := "text\n\n" + deferredBlock(notifyLine, []pick{conflict}, []pick{held}, nil) + "\n" + + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", body)}} + code, stdout, stderr := notifyRun(gh, "--mention", "@cozystack/maintainers", "--push") + if code != 0 || !strings.Contains(stdout, "release-1.1: 1 deferred conflicts on #90 wait for a maintainer") { + t.Fatalf("notify = %d, %q\n%s", code, stdout, stderr) + } + if len(gh.issueComments) != 1 { + t.Fatalf("%d comments, want one", len(gh.issueComments)) + } + c := gh.issueComments[0].Body + if !strings.Contains(c, "@cozystack/maintainers") || !strings.Contains(c, "[#6](") || !strings.Contains(c, "resolve-conflict.md") || strings.Contains(c, "#7") { + t.Errorf("the comment does not list #6 alone for a maintainer:\n%s", c) + } +} + +// Only a comment by an author the batch trusts is taken for the notify +// comment: a look-alike is neither edited nor counted. +func TestNotifyKeepsToItsOwnComment(t *testing.T) { + look := issueComment{ID: 7, Body: notifyMarker + "\nplanted"} + look.User.Login, look.User.Type = "someone", "User" + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}, issueComments: []issueComment{look}} + if code, _, stderr := notifyRun(gh, "--push"); code != 0 { + t.Fatalf("notify = %d\n%s", code, stderr) + } + if len(gh.issueComments) != 2 || gh.issueComments[0].Body != look.Body { + t.Errorf("comments = %+v; want the look-alike untouched and one of ours", gh.issueComments) + } + + // Ours says so once nothing waits any more, and calls nobody for it. + gh.prs = []pullRequest{notifyPR("20261026", "")} + if code, _, _ := notifyRun(gh, "--mention", "@cozystack/maintainers", "--push"); code != 0 || len(gh.issueComments) != 2 || + !strings.Contains(gh.issueComments[1].Body, "Nothing waits") || strings.Contains(gh.issueComments[1].Body, "@cozystack") { + t.Errorf("comments = %+v; want ours updated to say nothing waits, mentioning nobody", gh.issueComments) + } +} + +// The scheduled batch writes as its bot: notify keeps to the comment that bot +// wrote, on a pull request it opened, and to no other bot's. +func TestNotifyFindsTheBotsComment(t *testing.T) { + const bot = "cozystack-ci[bot]" + ours := issueComment{ID: 7, Body: notifyMarker + "\nearlier"} + ours.User.Login, ours.User.Type = bot, "Bot" + other := issueComment{ID: 8, Body: notifyMarker + "\nplanted"} + other.User.Login, other.User.Type = "renovate[bot]", "Bot" + // The bot writes the record into every body it opens. + pr := notifyPR("20261019", deferredBlock(notifyLine, nil, nil, nil)) + pr.User.Login, pr.User.Type = bot, "Bot" + gh := &fakeGH{prs: []pullRequest{pr}, issueComments: []issueComment{ours, other}, bot: bot} + if code, stdout, stderr := notifyRun(gh, "--push"); code != 0 || !strings.Contains(stdout, "has been open since") { + t.Fatalf("notify = %d, %q\n%s", code, stdout, stderr) + } + if len(gh.issueComments) != 2 || !strings.Contains(gh.issueComments[0].Body, "has been open since") || gh.issueComments[1].Body != other.Body { + t.Errorf("comments = %+v; want the bot's own comment edited and the other bot's untouched", gh.issueComments) + } + if code, _, stderr := notifyRun(&fakeGH{prs: []pullRequest{pr}, bot: "cozystack-ci"}, "--push"); code != 2 || !strings.Contains(stderr, batchBotEnv) { + t.Errorf("a %s that is no bot login = %d (%s), want a refusal", batchBotEnv, code, stderr) + } +} + +// A permission GitHub does not answer fails notify before it writes: read as +// a refusal, it would pass over a person's batch, or over the comment notify +// keeps and post a second one. +func TestNotifyFailsOnAnUnreadablePermission(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}, permErr: errString("HTTP 502")} + if code, stdout, stderr := notifyRun(gh, "--push"); code != 2 || stdout != "" || len(gh.issueComments) != 0 || !strings.Contains(stderr, "HTTP 502") { + t.Errorf("notify on a person's batch = %d, %q, %d comments (%s); want 2 and nothing written", code, stdout, len(gh.issueComments), stderr) + } + + const bot = "cozystack-ci[bot]" + pr := notifyPR("20261019", deferredBlock(notifyLine, nil, nil, nil)) + pr.User.Login, pr.User.Type = bot, "Bot" + ours := issueComment{ID: 7, Body: notifyMarker + "\nearlier"} + ours.User.Login, ours.User.Type = "maintainer", "User" + gh = &fakeGH{prs: []pullRequest{pr}, issueComments: []issueComment{ours}, bot: bot, permErr: errString("HTTP 502")} + if code, _, stderr := notifyRun(gh, "--push"); code != 2 || len(gh.issueComments) != 1 || gh.issueComments[0].Body != ours.Body || !strings.Contains(stderr, "HTTP 502") { + t.Errorf("notify with a person's comment = %d, comments %+v (%s); want 2 and the comment untouched", code, gh.issueComments, stderr) + } +} + +func TestNotifyRefuses(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}} + for _, args := range [][]string{ + {"--mention", "@a b"}, + {"--mention", "@org/team/x"}, + {"--mention", "@-x"}, + } { + if code, _, _ := notifyRun(gh, args...); code != 2 { + t.Errorf("notify %q = %d, want 2", args, code) + } + } + if code, stdout, _ := notifyRun(&fakeGH{}, "--push"); code != 0 || stdout != "" { + t.Errorf("notify with no batch open = %d, %q; want nothing", code, stdout) + } + pr := notifyPR("20261019", "") + pr.User.Login = "someone" + if code, stdout, _ := notifyRun(&fakeGH{prs: []pullRequest{pr}}, "--push"); code != 0 || stdout != "" { + t.Errorf("notify on a batch someone untrusted opened = %d, %q; want nothing", code, stdout) + } +} + +// notify reads the line's record the way the other commands do, and stays +// silent with no batch open, but a record it cannot read, or one that says +// its open batch is being closed, stops it. +func TestNotifyValidatesTheRecord(t *testing.T) { + sha := strings.Repeat("a", 40) + closingRecord := strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":91,"branch":"backport-batch/release-1.1-20261002","head":"`+sha+`"}`, 1) + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + for _, tc := range []struct { + name string + pr pullRequest + code int + quiet bool + }{ + {"a clean closed record", recordPR(91, "closed", "maintainer", block(v2(""))), 0, true}, + {"a malformed record", recordPR(91, "closed", "maintainer", block(`{"line":`)), 2, false}, + {"a closed record that was closing", recordPR(91, "closed", "maintainer", block(closingRecord)), 0, true}, + {"an open record that is closing", recordPR(91, "open", "maintainer", block(closingRecord)), 2, false}, + } { + gh := &fakeGH{prs: []pullRequest{tc.pr}} + code, stdout, stderr := notifyRun(gh, "--push") + if code != tc.code || tc.quiet && (stdout != "" || stderr != "") || len(gh.issueComments) != 0 { + t.Errorf("%s: notify = %d, %q, %q, %d comments; want %d and nothing written", tc.name, code, stdout, stderr, len(gh.issueComments), tc.code) + } + } +} + +// The maintainer's example: with 1.6 and 1.7 released, release-1.8 joins them +// when it branches, and 1.8.0 drops 1.6. +func TestSupportedLines(t *testing.T) { + l := func(minor int) line { return line{1, minor} } + cases := []struct { + name string + all []line + shipped []line + released int + want []line + }{ + {"two released", []line{l(5), l(6), l(7)}, []line{l(5), l(6), l(7)}, 2, []line{l(6), l(7)}}, + {"release-1.8 branched", []line{l(5), l(6), l(7), l(8)}, []line{l(5), l(6), l(7)}, 2, []line{l(6), l(7), l(8)}}, + {"1.8.0 shipped", []line{l(5), l(6), l(7), l(8)}, []line{l(5), l(6), l(7), l(8)}, 2, []line{l(7), l(8)}}, + {"three released", []line{l(5), l(6), l(7), l(8)}, []line{l(5), l(6), l(7)}, 3, []line{l(5), l(6), l(7), l(8)}}, + {"one released", []line{l(6), l(7), l(8)}, []line{l(6), l(7)}, 1, []line{l(7), l(8)}}, + {"a line between releases that never shipped", []line{l(5), l(6), l(7)}, []line{l(5), l(7)}, 2, []line{l(5), l(7)}}, + {"nothing shipped yet", []line{l(0), l(1)}, nil, 2, []line{l(0), l(1)}}, + } + for _, tc := range cases { + shipped := map[line]bool{} + for _, s := range tc.shipped { + shipped[s] = true + } + if got := supportedLines(tc.all, shipped, tc.released); !slices.Equal(got, tc.want) { + t.Errorf("%s: supportedLines = %v, want %v", tc.name, got, tc.want) + } + } +} + +// lines prints the lines the batch runs for, newest first, which is the order +// the scheduled run takes them in: the newest released lines, as many as the +// policy says, and a line branched since, whose stable release drops the +// oldest. +func TestLines(t *testing.T) { + f := newFixture(t) + f.release("release-0.9") + f.tag("v0.9.0") + f.lines() + f.release("release-1.2") + f.release("release-1.2.0-rc.1") + f.tag("v1.2.0-rc.1") + lines := func() string { + t.Helper() + code, stdout, stderr := f.exec(nil, "lines") + if code != 0 { + t.Fatalf("lines = %d (%s)", code, stderr) + } + return stdout + } + if got := lines(); got != "release-1.2\nrelease-1.1\nrelease-1.0\n" { + t.Errorf("with release-1.2 at its first rc, lines = %q; want it before the two released lines", got) + } + f.setPolicy(releasedKey + ": 3\n") + if got := lines(); got != "release-1.2\nrelease-1.1\nrelease-1.0\nrelease-0.9\n" { + t.Errorf("with three released lines, lines = %q; want release-0.9 too", got) + } + f.setPolicy("") + f.tag("v1.2.0") + if got := lines(); got != "release-1.2\nrelease-1.1\n" { + t.Errorf("once v1.2.0 ships, lines = %q; want release-1.0 dropped", got) + } + f.setPolicy(releasedKey + ": 3\n") + if got := lines(); got != "release-1.2\nrelease-1.1\nrelease-1.0\n" { + t.Errorf("once v1.2.0 ships, with three released lines, lines = %q", got) + } +} + +// A scheduled run with nothing to pick for a line succeeds without pushing or +// opening anything. +func TestRunWithNothingToPick(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "feat(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 0 || !strings.Contains(stderr, "nothing to push") { + t.Errorf("run with nothing to pick = %d, want 0:\n%s", code, stderr) + } + if f.remoteHead(batchBranch1) != "" || len(gh.created) != 0 { + t.Errorf("pushed %q and opened %q with nothing picked", f.remoteHead(batchBranch1), gh.created) + } +} + +// notify writes its comment only after GitHub says the batch is still open, +// so a batch closed while notify reads it is not written to, whether the +// comment would be new or an update of its own. +func TestNotifyWritesNothingOnABatchClosedMeanwhile(t *testing.T) { + for _, update := range []bool{false, true} { + t.Run(map[bool]string{false: "a new comment", true: "an update"}[update], func(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261023", recordBody(5))}} + if update { + ours := issueComment{ID: 7, Body: notifyMarker + "\nearlier"} + ours.User.Login, ours.User.Type = "maintainer", "User" + gh.issueComments = []issueComment{ours} + } + gh.after = map[string]func(){"comments": func() { gh.find(90).State = "closed" }} + code, _, stderr := notifyRun(gh, "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || len(gh.closedWrites) != 0 || update && gh.issueComments[0].Body != notifyMarker+"\nearlier" { + t.Errorf("notify = %d, closed writes %q, comments %+v; want 2 and nothing written:\n%s", code, gh.closedWrites, gh.issueComments, stderr) + } + }) + } +} diff --git a/cmd/backport-batch/onto_test.go b/cmd/backport-batch/onto_test.go new file mode 100644 index 000000000..ee4d52d33 --- /dev/null +++ b/cmd/backport-batch/onto_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "slices" + "strings" + "testing" +) + +// pushed builds and pushes a batch for #5 and returns its head. +func pushed(t *testing.T, f *fixture) string { + t.Helper() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + if code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + head := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + return head +} + +// --onto continues the pushed head and nothing else: it keeps what the batch +// carries and picks only what is new. +func TestOntoContinues(t *testing.T) { + f := newFixture(t) + f.lines() + head := pushed(t, f) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29") + if code != 2 || !strings.Contains(stderr, "not the head of backport-batch/release-1.1-20261029") { + t.Errorf("--onto with another date = %d (%s), want a refusal", code, stderr) + } + f.now = f.now.AddDate(0, 0, 3) + code, stdout, stderr = f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) + if code != 0 { + t.Fatalf("run --onto = %d\n%s", code, stderr) + } + if !strings.Contains(stderr, "title: fix(backport): batch for release-1.1, 2026-10-26\n") { + t.Errorf("the continued batch is not titled for the date it opened on:\n%s", stderr) + } + if got := f.subjects(head, "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q on top of the batch, want only #6", got) + } + if !descends(t, head, f.git("rev-parse", "HEAD")) { + t.Error("the new head does not descend from --onto") + } + if listed, _, _ := strings.Cut(stdout, "### Skipped"); !strings.Contains(listed, "[#5]") || !strings.Contains(listed, "[#6]") { + t.Errorf("the body does not list what the batch carries and what it picked:\n%s", stdout) + } +} + +// A head that is not the one the remote has now is refused, so work pushed +// since cannot be dropped. +func TestOntoRefusesAStaleHead(t *testing.T) { + f := newFixture(t) + f.lines() + head := pushed(t, f) + f.git("switch", "-q", "--detach", head) + f.write("app/fold.txt", "folded\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "fold") + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + + code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) + if code != 2 || !strings.Contains(stderr, "is not the head of any batch branch") { + t.Errorf("run --onto a stale head = %d (%s), want a refusal", code, stderr) + } + for _, bad := range []string{head[:12], "-" + head[1:], strings.ToUpper(head), "HEAD", "--help"} { + if code, _, _ := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", bad); code != 2 { + t.Errorf("--onto %q = %d, want 2", bad, code) + } + } +} + +// A rebuilt branch that does not descend from what the remote has is never +// pushed over it. +func TestPushRefusesANonFastForward(t *testing.T) { + f := newFixture(t) + f.lines() + head := pushed(t, f) + f.git("switch", "-q", "--detach", head) + f.write("app/fold.txt", "folded\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "fold") + folded := f.git("rev-parse", "HEAD") + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + + code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--push") + // git would refuse too, but only after trying; the tool refuses first. + if code != 2 || !strings.Contains(stderr, "refusing a non-fast-forward push") { + t.Errorf("a fresh rebuild pushed over the batch = %d (%s), want a refusal", code, stderr) + } + if got := f.remoteHead(batchBranch1); got != folded { + t.Errorf("the remote batch moved to %.12s, want it left at %.12s", got, folded) + } +} + +// A run that pushed its branch and stopped before opening the pull request +// leaves the branch behind. A run the same day builds the batch anew and +// refuses to push over it, naming the run that continues it, which then +// opens the pull request. +func TestOrphanBranchNamesItsRecovery(t *testing.T) { + f := newFixture(t) + f.lines() + head := pushed(t, f) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + // A commit picked again gets a new committer date, so a new name. + t.Setenv("GIT_COMMITTER_DATE", "2026-10-26T07:00:00Z") + code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--continue", "--push") + recovery := "backport-batch run --line release-1.1 --onto " + head + " --push" + if code != 2 || !strings.Contains(stderr, recovery) { + t.Fatalf("a same-day run over the left branch = %d (%s), want a refusal naming %q", code, stderr, recovery) + } + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, strings.Fields(recovery)[1:]...); code != 0 { + t.Fatalf("%s = %d\n%s", recovery, code, stderr) + } + now := f.remoteHead(batchBranch1) + if !descends(t, head, now) || now == head || len(gh.created) != 1 { + t.Errorf("the recovery pushed %.12s over %.12s and opened %q; want a fast-forward with #6 and one pull request", now, head, gh.created) + } +} diff --git a/cmd/backport-batch/overrides_test.go b/cmd/backport-batch/overrides_test.go new file mode 100644 index 000000000..0bcdc426e --- /dev/null +++ b/cmd/backport-batch/overrides_test.go @@ -0,0 +1,275 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// overridden builds a line where backport/ labels decide: #5 a feature let in, +// #6 a fix kept out, #7 a breaking fix and #8 a migration let in, #9 carrying +// both labels, #10 a feature let in that the line already carries, adapted, so +// picking it again would conflict rather than come out empty. +func overridden(t *testing.T) (*fixture, *fakeGH) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(5, "feat(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(app)!: seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.pr(8, "fix(platform): eight", "", map[string]string{migrationsDir + "61": "#!/bin/sh\n"}) + f.pr(9, "fix(app): nine", "", map[string]string{"app/nine.txt": "9\n"}) + mc10 := f.pr(10, "feat(app): ten", "", map[string]string{"app/ten.txt": "10\n"}) + f.onLine("release-1.1", "feat(app): ten (#10)\n\n(cherry picked from commit "+mc10+")", map[string]string{"app/ten.txt": "10, adapted\n"}) + gh := &fakeGH{labels: map[string][]int{ + labelInclude: {5, 7, 8, 9, 10}, + labelSkip: {6, 9}, + }} + return f, gh +} + +func TestOverrides(t *testing.T) { + f, gh := overridden(t) + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + want := []string{"feat(app): five (#5)", "fix(app)!: seven (#7)", "fix(platform): eight (#8)"} + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, want) { + t.Errorf("picked %q, want %q", got, want) + } + section, _, _ := strings.Cut(stdout[strings.Index(stdout, "### Maintainer overrides"):], "### Skipped") + for n, outcome := range map[int]string{ + 5: "`backport/include`, on this branch", + 6: "`backport/skip`, left out", + 7: "`backport/include`, on this branch", + 8: "`backport/include`, on this branch", + 9: "`backport/skip`, left out", + 10: "`backport/include`, skipped by rule 1", + } { + if !strings.Contains(section, fmt.Sprintf("/pull/%d)", n)) || !strings.Contains(section, outcome) { + t.Errorf("the overrides section does not show #%d as %q:\n%s", n, outcome, section) + } + } + if skipped := stdout[strings.Index(stdout, "### Skipped"):]; strings.Contains(skipped, "/pull/6)") || strings.Contains(skipped, "/pull/9)") { + t.Errorf("a skipped-by-label original is listed under a rule:\n%s", skipped) + } +} + +// check holds the line to what the labels decide, and fold refuses an +// original kept out. +func TestOverridesInCheckAndFold(t *testing.T) { + f, gh := overridden(t) + code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1") + if code != 1 { + t.Fatalf("check = %d, want 1:\n%s", code, stdout) + } + for _, n := range []int{5, 7, 8} { + if !strings.Contains(stdout, fmt.Sprintf("/pull/%d ", n)) { + t.Errorf("check does not list #%d:\n%s", n, stdout) + } + } + for _, n := range []int{6, 9, 10} { + if strings.Contains(stdout, fmt.Sprintf("/pull/%d ", n)) { + t.Errorf("check lists #%d:\n%s", n, stdout) + } + } + + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+batchBranch1) + f.git("fetch", "-q", "origin") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "")} + f.git("switch", "-q", "--detach", "origin/"+batchBranch1) + f.write("app/six.txt", "6\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "port\n\n(cherry picked from commit "+f.git("rev-parse", "origin/main~4")+")") + commit := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + code, stdout, _ = f.exec(gh, "fold", "--line", "release-1.1", "--original", "6", "--commit", commit) + if code != 1 || !strings.Contains(stdout, "#6 carries backport/skip") { + t.Errorf("fold of a skipped original = %d, %q; want a refusal", code, stdout) + } +} + +// backport/include lifts rules 2 to 4, not rule 6: an older line still waits +// for the newer one. +func TestIncludeKeepsRuleSix(t *testing.T) { + f, gh := overridden(t) + f.git("fetch", "-q", "origin") + l, _ := parseLine("release-1.0") + b, err := (&app{gh: gh, stderr: &strings.Builder{}}).prepare(l) + if err != nil { + t.Fatal(err) + } + if v := b.find(5); v == nil || v.rule != ruleNewer { + t.Errorf("#5 on release-1.0 = %+v, want rule 6", v) + } + if v := b.find(10); v == nil || v.rule != 0 { + t.Errorf("#10 on release-1.0 = %+v, want it to qualify: release-1.1 carries it", v) + } +} + +func TestOverridesUnreadable(t *testing.T) { + f, gh := overridden(t) + gh.listErr = errString("gh: HTTP 502") + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "HTTP 502") { + t.Errorf("check with labels unreadable = %d (%s), want 2", code, stderr) + } +} + +// ghCLI against a stub gh on PATH: the exact argv, and a listing that fills +// GitHub search's cap is refused rather than read as complete. +func TestGHCLI(t *testing.T) { + bin := t.TempDir() + args, out := filepath.Join(bin, "args"), filepath.Join(bin, "out") + stub := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"" + args + "\"\ncat \"" + out + "\"\n" + if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + write := func(s string) { + t.Helper() + if err := os.WriteFile(out, []byte(s), 0o644); err != nil { + t.Fatal(err) + } + _ = os.Remove(args) + } + called := func() string { + data, _ := os.ReadFile(args) + return strings.TrimSpace(string(data)) + } + + write(`[{"number":5},{"number":7}]`) + got, err := ghCLI{}.labelled(labelInclude) + if err != nil || !slices.Equal(got, []int{5, 7}) { + t.Errorf("labelled = %v, %v; want [5 7]", got, err) + } + if want := "pr list --base main --state merged --label backport/include --limit 1000 --json number"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + + var full []string + for i := range searchResultCap { + full = append(full, fmt.Sprintf(`{"number":%d}`, i+1)) + } + write("[" + strings.Join(full, ",") + "]") + if _, err := (ghCLI{}).labelled(labelSkip); err == nil { + t.Error("a listing as long as the search cap was read as complete") + } + + write(`{"permission":"write"}`) + if p, err := (ghCLI{}).permission("maintainer"); err != nil || p != "write" { + t.Errorf("permission = %q, %v", p, err) + } + if want := "api repos/{owner}/{repo}/collaborators/maintainer/permission"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write(`{"permission":"admin"}`) + for _, login := range []string{"x/../../orgs/y", "-x", "a b", "github-actions[bot]", ""} { + if _, err := (ghCLI{}).permission(login); err == nil || called() != "" { + t.Errorf("permission(%q) = %v and called gh %q; want a refusal before gh", login, err, called()) + } + } + + write("[]") + if _, err := (ghCLI{}).openPulls("release-1.6"); err != nil { + t.Fatal(err) + } + if want := "api -X GET repos/{owner}/{repo}/pulls -f state=open -f base=release-1.6 -f per_page=100"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write("[]") + if _, err := (ghCLI{}).recentPulls("release-1.6", 2); err != nil { + t.Fatal(err) + } + if want := "api -X GET repos/{owner}/{repo}/pulls -f state=all -f base=release-1.6 -f sort=created -f direction=desc -f per_page=100 -f page=2"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + var page []string + for i := range openPullsCap { + page = append(page, fmt.Sprintf(`{"number":%d}`, i+1)) + } + write("[" + strings.Join(page, ",") + "]") + if _, err := (ghCLI{}).openPulls("release-1.6"); err == nil { + t.Error("a full page of open pull requests was read as all of them") + } + + write(`[[{"id":1,"body":"a","user":{"login":"x","type":"User"}}],[{"id":2,"body":"b","user":{"login":"y","type":"Bot"}}]]`) + comments, err := ghCLI{}.comments(90) + if err != nil || len(comments) != 2 || comments[1].ID != 2 || comments[1].User.Type != "Bot" { + t.Errorf("comments = %+v, %v; want both pages", comments, err) + } + if want := "api --paginate --slurp repos/{owner}/{repo}/issues/90/comments"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write("{}") + if err := (ghCLI{}).createComment(90, "/tmp/body.md"); err != nil { + t.Fatal(err) + } + if want := "api --method POST repos/{owner}/{repo}/issues/90/comments -F body=@/tmp/body.md"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write("{}") + if err := (ghCLI{}).editComment(12345678901, "/tmp/body.md"); err != nil { + t.Fatal(err) + } + if want := "api --method PATCH repos/{owner}/{repo}/issues/comments/12345678901 -F body=@/tmp/body.md"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + + // A batch pull request carries no label: the release workflows act on a + // pull request cozystack-ci[bot] opened only when it has the release label. + write("https://github.com/cozystack/cozystack/pull/5300\n") + if n, url, err := (ghCLI{}).createPull("release-1.7", batchBranch1, "fix(backport): batch", "/tmp/body.md"); err != nil || n != 5300 || url != "https://github.com/cozystack/cozystack/pull/5300" { + t.Errorf("createPull = %d, %q, %v; want the pull request gh printed", n, url, err) + } + if want := "pr create --draft --base release-1.7 --head " + batchBranch1 + " --title fix(backport): batch --body-file /tmp/body.md"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write("Warning: 1 uncommitted change\n") + if _, _, err := (ghCLI{}).createPull("release-1.7", batchBranch1, "fix(backport): batch", "/tmp/body.md"); err == nil { + t.Error("createPull took output that is no pull request URL") + } + write(`{"number":90,"state":"closed","merged_at":"2026-10-26T12:00:00Z","head":{"ref":"x","sha":"y"}}`) + if pr, err := (ghCLI{}).pull(90); err != nil || pr.State != "closed" || pr.MergedAt == nil { + t.Errorf("pull = %+v, %v; want #90 merged", pr, err) + } + if want := "api repos/{owner}/{repo}/pulls/90"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + if _, err := (ghCLI{}).pull(91); err == nil { + t.Error("pull took GitHub's answer about another pull request") + } + write("") + if err := (ghCLI{}).closePull(90); err != nil { + t.Fatal(err) + } + if want := "pr close 90"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } + write("") + if err := (ghCLI{}).editPull(90, "fix(backport): batch", "/tmp/body.md"); err != nil { + t.Fatal(err) + } + if want := "pr edit 90 --title fix(backport): batch --body-file /tmp/body.md"; called() != want { + t.Errorf("gh %q, want gh %q", called(), want) + } +} diff --git a/cmd/backport-batch/policy.go b/cmd/backport-batch/policy.go new file mode 100644 index 000000000..ee74c76e8 --- /dev/null +++ b/cmd/backport-batch/policy.go @@ -0,0 +1,144 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// policyPage is the policy the batch applies. The block in it that names when +// each line started under the policy is read from the page itself, on the ref +// the run pins, so the page and the batch cannot drift apart. +const policyPage = "docs/release/backports.md" + +const policyFence = "```backport-policy" + +var ( + policyLineRE = regexp.MustCompile(`^(release-\d+\.\d+|` + batchOnlyKey + `):[ \t]*(\d{4}-\d{2}-\d{2})[ \t]*$`) + policyReleasedRE = regexp.MustCompile(`^` + releasedKey + `:[ \t]*(\d+)[ \t]*$`) +) + +const ( + // batchOnlyKey names the day from which the batch is the only way onto a + // line, so that from then on only its picks show what a line carries. + batchOnlyKey = "batch-only" + // releasedKey names how many of the lines that have shipped a stable + // release the batch runs for, the newest of them; defaultReleased when + // the block does not say. + releasedKey = "released-lines" + defaultReleased = 2 +) + +// The rule 2 types and the scopes that never go into a patch release, as the +// policy page lists them; policy_test.go holds the page to these. +var ( + takenTypes = []string{"fix", "security"} + excludedScopes = []string{"deps"} +) + +// starts maps a release line to the day it started under the policy. A line +// that is not listed started at its fork point from main. +type starts map[string]time.Time + +// policy is what the backport-policy block says: when each line started, the +// batch-only day, and how many released lines the batch runs for. +type policy struct { + starts starts + batchOnly time.Time + released int +} + +// parsePolicy reads the one backport-policy block of the page: a line per +// release line, `release-X.Y: YYYY-MM-DD`, exactly one `batch-only: +// YYYY-MM-DD`, at most one `released-lines: N`, and comments starting with #. +func parsePolicy(page string) (*policy, error) { + var blocks [][]string + var cur []string + in := false + for _, ln := range strings.Split(strings.ReplaceAll(page, "\r\n", "\n"), "\n") { + t := strings.TrimSpace(ln) + switch { + case !in && t == policyFence: + in, cur = true, nil + case in && t == "```": + in = false + blocks = append(blocks, cur) + case in: + cur = append(cur, t) + } + } + if in { + return nil, fmt.Errorf("%s: the %s block is not closed", policyPage, policyFence) + } + if len(blocks) != 1 { + return nil, fmt.Errorf("%s holds %d %s blocks, not one", policyPage, len(blocks), policyFence) + } + out := &policy{starts: starts{}, released: defaultReleased} + seen := map[string]bool{} + for _, t := range blocks[0] { + if t == "" || strings.HasPrefix(t, "#") { + continue + } + if m := policyReleasedRE.FindStringSubmatch(t); m != nil { + n, err := strconv.Atoi(m[1]) + switch { + case err != nil || n < 1: + return nil, fmt.Errorf("%s: %s is %q, not a number of lines", policyPage, releasedKey, m[1]) + case seen[releasedKey]: + return nil, fmt.Errorf("%s: %s is listed twice", policyPage, releasedKey) + } + seen[releasedKey], out.released = true, n + continue + } + m := policyLineRE.FindStringSubmatch(t) + if m == nil { + return nil, fmt.Errorf("%s: %q in the %s block is not `release-X.Y: YYYY-MM-DD`, `%s: YYYY-MM-DD` or `%s: N`", policyPage, t, policyFence, batchOnlyKey, releasedKey) + } + day, err := time.Parse(time.DateOnly, m[2]) + if err != nil { + return nil, fmt.Errorf("%s: %q is not a date: %w", policyPage, m[2], err) + } + if seen[m[1]] { + return nil, fmt.Errorf("%s: %s is listed twice", policyPage, m[1]) + } + seen[m[1]] = true + if m[1] == batchOnlyKey { + out.batchOnly = day + continue + } + if _, err := parseLine(m[1]); err != nil { + return nil, fmt.Errorf("%s: %w", policyPage, err) + } + out.starts[m[1]] = day + } + if !seen[batchOnlyKey] { + return nil, fmt.Errorf("%s: the %s block gives no %s day", policyPage, policyFence, batchOnlyKey) + } + return out, nil +} + +func readPolicy(commit string) (*policy, error) { + page, err := git("show", "--end-of-options", commit+":"+policyPage) + if err != nil { + return nil, fmt.Errorf("reading the policy at %.12s: %w", commit, err) + } + return parsePolicy(page) +} diff --git a/cmd/backport-batch/policy_test.go b/cmd/backport-batch/policy_test.go new file mode 100644 index 000000000..920c18a0b --- /dev/null +++ b/cmd/backport-batch/policy_test.go @@ -0,0 +1,212 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "regexp" + "slices" + "strings" + "testing" + "time" +) + +func TestParsePolicy(t *testing.T) { + got, err := parsePolicy("text\n\n" + policyFence + "\n# comment\n\nrelease-1.6: 2026-10-26\nrelease-1.5:2026-01-02\nbatch-only: 2026-10-20\n```\n") + if err != nil { + t.Fatal(err) + } + if len(got.starts) != 2 || !got.starts["release-1.6"].Equal(time.Date(2026, 10, 26, 0, 0, 0, 0, time.UTC)) { + t.Errorf("parsePolicy = %v", got) + } + if !got.batchOnly.Equal(time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC)) { + t.Errorf("the batch-only day = %v, want 2026-10-20", got.batchOnly) + } + if got.released != 2 { + t.Errorf("released lines with none given = %d, want 2", got.released) + } + if got, err := parsePolicy(policyFence + "\nbatch-only: 2026-10-20\n```\n"); err != nil || len(got.starts) != 0 { + t.Errorf("a block with no line = %v, %v; want no starts", got, err) + } + if got, err := parsePolicy(policyFence + "\nbatch-only: 2026-10-20\nreleased-lines: 3\n```\n"); err != nil || got.released != 3 { + t.Errorf("released-lines: 3 = %v, %v; want 3", got, err) + } + for name, page := range map[string]string{ + "no batch-only day": policyFence + "\nrelease-1.6: 2026-10-26\n```\n", + "two batch-only days": policyFence + "\nbatch-only: 2026-10-26\nbatch-only: 2026-10-27\n```\n", + "a bad batch-only day": policyFence + "\nbatch-only: soon\n```\n", + "no block": "## Backports\n", + "two blocks": policyFence + "\n```\n" + policyFence + "\n```\n", + "unclosed": policyFence + "\nrelease-1.6: 2026-10-26\n", + "a bad line": policyFence + "\nrelease 1.6 on 2026-10-26\n```\n", + "a bad date": policyFence + "\nrelease-1.6: 2026-13-40\n```\n", + "a padded line": policyFence + "\nrelease-01.6: 2026-10-26\n```\n", + "listed twice": policyFence + "\nrelease-1.6: 2026-10-26\nrelease-1.6: 2026-10-27\n```\n", + "not a line": policyFence + "\nmain: 2026-10-26\n```\n", + "trailing words": policyFence + "\nrelease-1.6: 2026-10-26 or so\n```\n", + "no released lines": policyFence + "\nbatch-only: 2026-10-26\nreleased-lines: 0\n```\n", + "released lines twice": policyFence + "\nbatch-only: 2026-10-26\nreleased-lines: 2\nreleased-lines: 3\n```\n", + "released lines words": policyFence + "\nbatch-only: 2026-10-26\nreleased-lines: two\n```\n", + "released lines huge": policyFence + "\nbatch-only: 2026-10-26\nreleased-lines: 99999999999999999999\n```\n", + } { + if _, err := parsePolicy(page); err == nil { + t.Errorf("%s: parsePolicy accepted it", name) + } + } +} + +// A line listed in the policy takes nothing merged before its start, unless a +// maintainer lets an original in; any other line starts at its fork point. +func TestPolicyStart(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "feat(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{labels: map[string][]int{labelInclude: {6, 7}}} + + verdicts := func() map[int]*verdict { + t.Helper() + f.git("fetch", "-q", "origin") + l, _ := parseLine("release-1.1") + b, err := (&app{gh: gh, stderr: &strings.Builder{}}).prepare(l) + if err != nil { + t.Fatal(err) + } + return map[int]*verdict{5: b.find(5), 6: b.find(6), 7: b.find(7)} + } + if v := verdicts(); v[5].rule != 0 || v[6].rule != 0 || v[7].rule != 0 { + t.Errorf("with no start every original qualifies: %d %d %d", v[5].rule, v[6].rule, v[7].rule) + } + + f.setPolicy("release-1.1: 2099-01-01\n") + v := verdicts() + if v[5].rule != ruleBeforeStart { + t.Errorf("#5 = rule %d (%s), want merged before the start", v[5].rule, v[5].reason) + } + if v[6].rule != 0 || v[7].rule != 0 { + t.Errorf("backport/include lifts the start: #6 = rule %d, #7 = rule %d", v[6].rule, v[7].rule) + } + + f.setPolicy("release-1.0: 2099-01-01\n") + if v := verdicts(); v[5].rule != 0 { + t.Errorf("a start for another line kept #5 out of release-1.1: rule %d", v[5].rule) + } + + f.setPolicy("release-1.1: 2000-01-01\n") + if v := verdicts(); v[5].rule != 0 { + t.Errorf("a start before the fork point kept #5 out: rule %d", v[5].rule) + } +} + +// The policy is read from main as the run pins it, or from the ref --policy +// names, and a run without a readable policy stops. Only the commands that +// read the policy take --policy. +func TestPolicyRef(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.git("switch", "-q", "-c", "proposal") + f.write(policyPage, policyWith("release-1.1: 2099-01-01\n"+releasedKey+": 1\n")) + f.git("commit", "-q", "-am", "docs: start release-1.1 later") + f.git("switch", "-q", "main") + + code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1") + if code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check against main's policy = %d, %q, %s; want #5 missing", code, stdout, stderr) + } + code, stdout, stderr = f.exec(nil, "check", "--line", "release-1.1", "--policy", "proposal") + if code != 0 { + t.Errorf("check against the proposal's policy = %d, %q, %s; want #5 before the start", code, stdout, stderr) + } + if code, _, _ := f.exec(nil, "check", "--line", "release-1.1", "--policy", "--upload-pack=x"); code != 2 { + t.Errorf("--policy with an option = %d, want 2", code) + } + if code, stdout, stderr := f.exec(nil, "lines", "--policy", "proposal"); code != 0 || stdout != "release-1.1\n" { + t.Errorf("lines against the proposal's policy = %d, %q (%s); want release-1.1 alone", code, stdout, stderr) + } + for _, cmd := range []string{"verify", "notify"} { + if code, _, stderr := f.exec(&fakeGH{}, cmd, "--line", "release-1.1", "--policy", "proposal"); code != 2 || !strings.Contains(stderr, "flag provided but not defined: -policy") { + t.Errorf("%s --policy = %d (%s); want it refused, as %s reads no policy", cmd, code, stderr, cmd) + } + } + + f.squash(9001, "docs: drop the page", "", map[string]string{policyPage: ""}) + if code, _, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "reading the policy") { + t.Errorf("check without a policy page = %d (%s), want 2", code, stderr) + } +} + +// The policy page and the code say the same thing: the start block parses as +// the batch reads it, rule 2 names the types the code takes, the page names +// the scopes the code keeps out, and both override labels. +func TestPolicyPageMatchesCode(t *testing.T) { + data, err := os.ReadFile("../../" + policyPage) + if err != nil { + t.Fatal(err) + } + page := string(data) + pol, err := parsePolicy(page) + if err != nil { + t.Fatalf("the page's policy block does not parse: %v", err) + } + if day, ok := pol.starts["release-1.6"]; !ok || day.Format(time.DateOnly) != "2026-10-26" { + t.Errorf("release-1.6 starts %v, want 2026-10-26", pol.starts["release-1.6"]) + } + if day := pol.batchOnly.Format(time.DateOnly); day != "2026-10-26" { + t.Errorf("the batch-only day is %s, want 2026-10-26", day) + } + // The page says how many released lines the batch runs for, rather than + // leaving it to the default. + if !regexp.MustCompile("(?m)^"+releasedKey+":").MatchString(page) || pol.released != 2 { + t.Errorf("the page's block gives %d released lines, want 2 written out", pol.released) + } + + rule2 := regexp.MustCompile("(?m)^2\\. Its title type is (.*)$").FindStringSubmatch(page) + if rule2 == nil { + t.Fatal("the page has no rule 2") + } + var types []string + for _, m := range regexp.MustCompile("`([a-z]+)`").FindAllStringSubmatch(rule2[1], -1) { + types = append(types, m[1]) + } + if !slices.Equal(types, takenTypes) { + t.Errorf("rule 2 on the page takes %v, the code %v", types, takenTypes) + } + never := regexp.MustCompile("(?m)^### What waits for the next minor\\n\\n(.*)$").FindStringSubmatch(page) + if never == nil { + t.Fatal("the page has no list of what never goes into a patch release") + } + for _, scope := range excludedScopes { + if !strings.Contains(never[1], "(scope `"+scope+"`)") { + t.Errorf("the page does not keep scope %s out", scope) + } + } + if scopes := regexp.MustCompile("scope `([a-z]+)`").FindAllStringSubmatch(never[1], -1); len(scopes) != len(excludedScopes) { + t.Errorf("the page keeps %d scopes out, the code %d", len(scopes), len(excludedScopes)) + } + for i := 1; i <= 6; i++ { + if !regexp.MustCompile("(?m)^" + string(rune('0'+i)) + "\\. ").MatchString(page) { + t.Errorf("the page has no rule %d", i) + } + } + for _, label := range []string{labelInclude, labelSkip} { + if !strings.Contains(page, "`"+label+"`") { + t.Errorf("the page does not describe %s", label) + } + } +} diff --git a/cmd/backport-batch/record.go b/cmd/backport-batch/record.go new file mode 100644 index 000000000..4ab43933e --- /dev/null +++ b/cmd/backport-batch/record.go @@ -0,0 +1,440 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strings" + "time" +) + +// A line's record is the backport-batch block of its newest batch pull +// request, read whatever that pull request's state and written only while it +// is open: a merged or closed pull request is never written again, so what it +// recorded stays what the line's later commands act on until a newer batch +// carries a record of its own. + +// lineRecord is the one line of JSON in the block. Version 1, the block +// before it had a version, holds only line and deferred. +type lineRecord struct { + Version int `json:"version"` + Line string `json:"line"` + Built *builtFrom `json:"built,omitempty"` + Report string `json:"report,omitempty"` + Closing *closing `json:"closing"` + Deferred []deferredEntry `json:"deferred"` + Dropped []droppedEntry `json:"dropped"` + Stale []int `json:"stale"` +} + +// builtFrom is what a classification was made of; batch is the head it +// describes. +type builtFrom struct { + Main string `json:"main"` + Line string `json:"line"` + Newer string `json:"newer"` + Policy string `json:"policy"` + Batch string `json:"batch"` + Tool string `json:"tool"` + Run string `json:"run"` + At string `json:"at"` + Options runOptions `json:"options"` +} + +type runOptions struct { + NoWorkflowFiles bool `json:"noWorkflowFiles"` + Mode string `json:"mode"` +} + +// The operations a record is written by: a run that assembles the batch, and +// the ones that only change what the record says about it. +const ( + modeNew = "new" + modeContinue = "continue" + modeDefer = "defer" + modeDrop = "drop" + modeUndrop = "undrop" + modeClose = "close" +) + +var modes = []string{modeNew, modeContinue, modeDefer, modeDrop, modeUndrop, modeClose} + +// closing is written into the record of an open batch that the run is about +// to close: the head it froze, which the close and the branch's deletion are +// leased on. +type closing struct { + PR int `json:"pr"` + Branch string `json:"branch"` + Head string `json:"head"` +} + +// droppedEntry is a maintainer's decision that an original stays off the +// line, bound to its merge commit on main. +type droppedEntry struct { + Original int `json:"original"` + Merge string `json:"merge"` + Reason string `json:"reason"` + On int `json:"on"` + Date string `json:"date"` +} + +var ( + toolRE = regexp.MustCompile(`^[0-9a-f]{40}(\+dirty)?$`) + reportIDRE = regexp.MustCompile(`^[0-9a-f]{64}$`) +) + +// oneLineText is s as a run writes a reason: one line, cleaned and capped. +// A record holding anything else was not written by a run, and its text +// would reach a terminal, a body or a comment as someone else wrote it. +func oneLineText(s string) string { return capped(oneLine(clean(s, false)), titleLimit) } + +// fencedLines lists the one line of each block that fence opens. +func fencedLines(body, fence string) ([]string, error) { + var blocks []string + lines := strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") + for i := 0; i < len(lines); i++ { + if strings.TrimSpace(lines[i]) != fence { + continue + } + if i+2 >= len(lines) || strings.TrimSpace(lines[i+2]) != "```" { + return nil, fmt.Errorf("the %s block is not one line of JSON", fence) + } + blocks = append(blocks, lines[i+1]) + i += 2 + } + return blocks, nil +} + +// decodeStrict reads exactly one JSON value into v, refusing a key v does not +// have and anything after the value. +func decodeStrict(data string, v any) error { + dec := json.NewDecoder(strings.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + var rest json.RawMessage + if dec.More() || dec.Decode(&rest) != io.EOF { + return errors.New("something follows the JSON value") + } + return nil +} + +// parseRecord reads the line's record from a batch body: nil when the body +// has no block, an error when it has more than one or one it cannot vouch +// for whole. +func parseRecord(body string, l line) (*lineRecord, error) { + blocks, err := fencedLines(body, deferredFence) + switch { + case err != nil: + return nil, err + case len(blocks) == 0: + return nil, nil + case len(blocks) > 1: + return nil, fmt.Errorf("the body holds %d %s blocks", len(blocks), deferredFence) + } + var head struct { + Version *int `json:"version"` + } + if err := json.Unmarshal([]byte(blocks[0]), &head); err != nil { + return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) + } + version := 1 + if head.Version != nil { + version = *head.Version + } + var rec lineRecord + switch version { + case 1: + var v1 struct { + Version int `json:"version"` + Line string `json:"line"` + Deferred []deferredEntry `json:"deferred"` + } + err = decodeStrict(blocks[0], &v1) + rec = lineRecord{Version: 1, Line: v1.Line, Deferred: v1.Deferred} + case 2: + err = decodeStrict(blocks[0], &rec) + default: + return nil, fmt.Errorf("the %s block is version %d, which this backport-batch does not read; it reads 1 and 2", deferredFence, version) + } + if err != nil { + return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) + } + if err := rec.validate(l); err != nil { + return nil, fmt.Errorf("the %s block %w", deferredFence, err) + } + return &rec, nil +} + +// validate holds the record to what a run writes. Its errors read after +// "the block". +func (rec *lineRecord) validate(l line) error { + if rec.Line != l.branch() { + return fmt.Errorf("is for %q, not %s", rec.Line, l.branch()) + } + seen := map[int]bool{} + once := func(n int) error { + if n <= 0 { + return fmt.Errorf("names original %d", n) + } + if seen[n] { + return fmt.Errorf("names #%d twice", n) + } + seen[n] = true + return nil + } + for _, d := range rec.Deferred { + if err := once(d.Original); err != nil { + return err + } + if d.Commit != "" && checkSHA(d.Commit) != nil { + return fmt.Errorf("names commit %q for #%d", d.Commit, d.Original) + } + // A conflict names the commit that failed and its files, a deferral + // its reason, and a wait its workflow files. + kinds := 0 + for _, is := range []bool{d.Commit != "" || len(d.Files) > 0 && !d.Workflows && d.Reason == "", d.Reason != "", d.Workflows} { + if is { + kinds++ + } + } + switch { + case d.Workflows && (d.Commit != "" || d.Reason != ""): + return fmt.Errorf("has #%d wait for a maintainer and defers it as well", d.Original) + case kinds != 1: + return fmt.Errorf("does not say whether #%d is a conflict, deferred for a reason or waiting", d.Original) + case d.Reason != "" && strings.TrimSpace(d.Reason) == "": + return fmt.Errorf("defers #%d for an empty reason", d.Original) + case d.Reason != oneLineText(d.Reason): + return fmt.Errorf("defers #%d for %q, which is no reason a run writes", d.Original, d.Reason) + case !slices.Equal(d.Files, cleanFiles(d.Files)): + return fmt.Errorf("names files of #%d that are not as a run writes them: %q", d.Original, d.Files) + case d.More < 0: + return fmt.Errorf("counts %d more files for #%d", d.More, d.Original) + } + } + for _, d := range rec.Dropped { + if err := once(d.Original); err != nil { + return err + } + if _, err := time.Parse(time.DateOnly, d.Date); err != nil || checkSHA(d.Merge) != nil || strings.TrimSpace(d.Reason) == "" || d.On <= 0 { + return fmt.Errorf("drops #%d with a merge, reason, pull request or date it cannot have", d.Original) + } + if d.Reason != oneLineText(d.Reason) { + return fmt.Errorf("drops #%d for %q, which is no reason a run writes", d.Original, d.Reason) + } + } + staleSeen := map[int]bool{} + for _, n := range rec.Stale { + if n <= 0 || staleSeen[n] { + return fmt.Errorf("lists %d as still on the batch more than once or not at all", n) + } + staleSeen[n] = true + } + if rec.Version < 2 { + return nil + } + if b := rec.Built; b == nil { + return errors.New("does not say what it was built from") + } else if err := b.validate(); err != nil { + return err + } + if !reportIDRE.MatchString(rec.Report) { + return fmt.Errorf("names report %q", rec.Report) + } + if c := rec.Closing; c != nil { + m := batchBranchRE.FindStringSubmatch(c.Branch) + if c.PR <= 0 || m == nil || m[1] != l.branch() || checkSHA(c.Head) != nil { + return fmt.Errorf("closes #%d on %q at %q, which is no batch of %s", c.PR, c.Branch, c.Head, l.branch()) + } + } + return nil +} + +func (b *builtFrom) validate() error { + for _, sha := range []string{b.Main, b.Line, b.Policy, b.Batch} { + if checkSHA(sha) != nil { + return fmt.Errorf("says it was built from %q", sha) + } + } + if b.Newer != "" && checkSHA(b.Newer) != nil { + return fmt.Errorf("says it was built from %q", b.Newer) + } + if b.Tool != "" && !toolRE.MatchString(b.Tool) { + return fmt.Errorf("names tool %q", b.Tool) + } + if b.Run != "" && !runURLRE.MatchString(b.Run) { + return fmt.Errorf("names run %q", b.Run) + } + if _, err := time.Parse(time.RFC3339, b.At); err != nil { + return fmt.Errorf("was built at %q", b.At) + } + if !slices.Contains(modes, b.Options.Mode) { + return fmt.Errorf("was written by %q, which is no operation of backport-batch", b.Options.Mode) + } + return nil +} + +var runURLRE = regexp.MustCompile(`^https://[^/\s]+/[^/\s]+/[^/\s]+/actions/runs/[0-9]+(/attempts/[0-9]+)?$`) + +// recordBlock is the block that holds rec. One line of JSON: encoding/json +// escapes newlines and <, > and &, so nothing in a file name or a reason can +// end the fence or open HTML. +func recordBlock(rec lineRecord) string { + data, _ := json.Marshal(rec) + return deferredFence + "\n" + string(data) + "\n```" +} + +// savedNoWorkflowFiles is the --no-workflow-files the open batch was last +// assembled with, which an operation that only changes the record keeps. A +// record from before it was saved says so by listing a wait. +func (s recordSource) savedNoWorkflowFiles() bool { + if !s.open || s.rec == nil { + return false + } + if s.rec.Built != nil { + return s.rec.Built.Options.NoWorkflowFiles + } + return slices.ContainsFunc(s.rec.Deferred, func(d deferredEntry) bool { return d.Workflows }) +} + +// dropped is what the record drops, which every later record keeps. +func (rec *lineRecord) dropped() []droppedEntry { + if rec == nil { + return nil + } + return slices.Clone(rec.Dropped) +} + +// deferred maps the originals the record defers to their entries. +func (rec *lineRecord) deferred() map[int]deferredEntry { + out := map[int]deferredEntry{} + if rec != nil { + for _, d := range rec.Deferred { + out[d.Original] = d + } + } + return out +} + +// recordSource is where a line's record was found. +type recordSource struct { + // carrier is the pull request whose block is the record, nil when the + // line has none. + carrier *pullRequest + // open says the carrier is the open batch, so the record may be written. + open bool + rec *lineRecord + // code is the open batch pull request, if any. One whose body someone + // wrote by hand is the batch the line builds on but carries no record. + code *pullRequest +} + +// closingOpen is the close the record says is under way on a batch that is +// still open, or nil. +func (s recordSource) closingOpen() *closing { + if !s.open || s.rec == nil { + return nil + } + return s.rec.Closing +} + +func prState(pr *pullRequest) string { + switch { + case pr.MergedAt != nil: + return "merged" + case pr.State == "" || pr.State == "open": + return "open" + case pr.State == "closed": + return "closed" + } + return fmt.Sprintf("in state %q", pr.State) +} + +func hasRecord(body string) bool { + return slices.ContainsFunc(strings.Split(body, "\n"), func(ln string) bool { return strings.TrimSpace(ln) == deferredFence }) +} + +// findLineRecord finds the line's record: the block of the newest pull +// request opened from a batch branch of this repository for the line by an +// author the batch trusts, open, merged or closed. open is the open batch +// pull request batchPull chose, or nil. Anything that leaves in doubt which +// record is the newest is an error, never a fall back to an older one: a +// newer block by an author the batch does not trust, a block the bot wrote +// and someone removed, a record it cannot read, a closed record newer than +// the open batch, or more pull requests than it reads. +func findLineRecord(gh github, l line, open *pullRequest) (recordSource, error) { + src := recordSource{code: open} + for page := 1; src.carrier == nil; page++ { + if page > recentPullsPages { + return src, fmt.Errorf("none of the %d newest pull requests against %s is a batch pull request with a record, so the line's record cannot be told apart from none", recentPullsPages*recentPullsPage, l.branch()) + } + prs, err := gh.recentPulls(l.branch(), page) + if err != nil { + return src, err + } + for i := range prs { + pr := &prs[i] + m := batchBranchRE.FindStringSubmatch(pr.Head.Ref) + if m == nil || m[1] != l.branch() || pr.Base.Ref != l.branch() || pr.Head.Repo == nil || pr.Head.Repo.FullName != pr.Base.Repo.FullName { + continue + } + if open != nil && pr.Number == open.Number { + pr = open + } + if !hasRecord(pr.Body) { + if pr.User.Type == "Bot" && gh.batchBot() != "" && pr.User.Login == gh.batchBot() { + return src, fmt.Errorf("batch pull request #%d (%s) was opened by %s but carries no %s block; someone removed it, so the line's record is unknown", pr.Number, prState(pr), pr.User.Login, deferredFence) + } + continue + } + why, err := trusted(gh, *pr) + if err != nil { + return src, fmt.Errorf("batch pull request #%d: %w", pr.Number, err) + } + if why != "" { + return src, fmt.Errorf("batch pull request #%d (%s) carries a %s block but was %s; an older record is not taken in its place", pr.Number, prState(pr), deferredFence, why) + } + rec, err := parseRecord(pr.Body, l) + if err != nil { + return src, fmt.Errorf("batch pull request #%d (%s): %w", pr.Number, prState(pr), err) + } + src.carrier, src.rec = pr, rec + break + } + if len(prs) < recentPullsPage { + break + } + } + switch c := src.carrier; { + case open != nil && hasRecord(open.Body) && (c == nil || c.Number < open.Number): + return src, fmt.Errorf("GitHub does not list the open batch pull request #%d among the pull requests against %s; run again", open.Number, l.branch()) + case open != nil && hasRecord(open.Body) && c.Number > open.Number: + return src, fmt.Errorf("batch pull request #%d (%s) carries a record newer than the open batch pull request #%d; close one of them by hand", c.Number, prState(c), open.Number) + case open != nil && c == open: + src.open = true + case c != nil && prState(c) == "open": + return src, fmt.Errorf("batch pull request #%d carries the line's record and is open, but is not the batch this run builds on", c.Number) + } + return src, nil +} diff --git a/cmd/backport-batch/record_test.go b/cmd/backport-batch/record_test.go new file mode 100644 index 000000000..13ab6cc79 --- /dev/null +++ b/cmd/backport-batch/record_test.go @@ -0,0 +1,368 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + "testing" +) + +// The records the live batches carried before records had a version, as +// GitHub returned them on 2026-11-09, read as version 1. +func TestRecordReadsVersion1(t *testing.T) { + for line, block := range map[string]string{ + "release-1.7": `{"line":"release-1.7","deferred":[{"original":4756,"commit":"647407545d630291cfe2a20880cce45af22edf0c","files":["packages/system/backupstrategy-controller/values.yaml"]},{"original":4867,"commit":"813c6180861efe1896c26d62a7afcbc1ff590ebc","files":[".github/renovate.json"]},{"original":5082,"commit":"e93db8848027450c2e23d21348409efa641cb6f1","files":["pkg/registry/core/tap/rest.go"]},{"original":4888,"files":[".github/workflows/cut-prerelease.yaml"],"workflows":true},{"original":4971,"files":[".github/workflows/e2e-proxmox.yaml"],"workflows":true}]}`, + "release-1.6": `{"line":"release-1.6","deferred":[{"original":5071,"commit":"3fa534f3cf6cb5774a183f0202f7f5c13e9ae211","files":["internal/operator/tapmaterializer_reconciler.go","internal/operator/tapmaterializer_reconciler_test.go"]},{"original":5086,"commit":"5265ee662c591db409c8ef1ed8bb457cf29f5c22","files":["packages/apps/tenant/templates/gateway.yaml"]},{"original":5087,"commit":"8d8179f20a2c10de03face1472435d5083cde38d","files":["internal/controller/applicationdefinition_helmreconciler.go","internal/controller/applicationdefinition_helmreconciler_test.go"]},{"original":5142,"files":[".github/workflows/pull-requests-release.yaml",".github/workflows/update-releasenotes.yaml"],"workflows":true}]}`, + } { + l, _ := parseLine(line) + rec, err := parseRecord("## What this PR does\n\ntext\n\n"+deferredFence+"\n"+block+"\n```\n\n### Release note\n", l) + if err != nil { + t.Fatalf("%s: parseRecord = %v", line, err) + } + waits, conflicts := 0, 0 + for _, e := range rec.Deferred { + switch { + case e.Workflows: + waits++ + case e.Commit != "": + conflicts++ + } + } + if rec.Version != 1 || rec.Line != line || rec.Closing != nil || len(rec.Dropped) != 0 || waits+conflicts != len(rec.Deferred) || waits == 0 || conflicts == 0 { + t.Errorf("%s: record = %+v; want version 1 with its conflicts and waits", line, rec) + } + } +} + +// v2 is a version 2 record with each value it may hold. +func v2(extra string) string { + sha := strings.Repeat("a", 40) + return `{"version":2,"line":"release-1.1","built":{"main":"` + sha + `","line":"` + sha + `","newer":"","policy":"` + sha + `","batch":"` + sha + `","tool":"` + sha + `+dirty","run":"","at":"2026-10-26T06:00:00Z","options":{"noWorkflowFiles":true,"mode":"continue"}},` + + `"report":"` + strings.Repeat("b", 64) + `","closing":null,"deferred":[{"original":5,"commit":"` + sha + `","files":["app/five.txt"]},{"original":6,"files":[],"reason":"fails the e2e"},{"original":7,"files":[".github/workflows/x.yaml"],"workflows":true}],` + + `"dropped":[{"original":8,"merge":"` + sha + `","reason":"needs #4","on":90,"date":"2026-10-26"}],"stale":[9]` + extra + `}` +} + +// A record is read whole or not at all: anything it cannot vouch for stops +// the command rather than being half believed. +func TestRecordRefuses(t *testing.T) { + l, _ := parseLine("release-1.1") + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + if rec, err := parseRecord(block(v2("")), l); err != nil || rec.Version != 2 || len(rec.Dropped) != 1 || rec.Built.Options.Mode != "continue" { + t.Fatalf("parseRecord of a whole v2 record = %+v, %v", rec, err) + } + sha := strings.Repeat("a", 40) + for name, json := range map[string]string{ + "version 4": strings.Replace(v2(""), `"version":2`, `"version":4`, 1), + "trailing data": v2("") + `{}`, + "trailing text": v2("") + ` x`, + "an original listed twice": strings.Replace(v2(""), `"original":8`, `"original":5`, 1), + "a needs entry with a commit": strings.Replace(v2(""), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"commit":"`+sha+`","files":[],"needs":[4]}`, 1), + "an empty reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":""`, 1), + "an empty drop reason": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":" "`, 1), + "a bad tool": strings.Replace(v2(""), `+dirty"`, `-dirty"`, 1), + "an unknown mode": strings.Replace(v2(""), `"mode":"continue"`, `"mode":"rebase"`, 1), + "a reason with a commit": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"x","commit":"`+sha+`"`, 1), + "a short merge": strings.Replace(v2(""), `"merge":"`+sha+`"`, `"merge":"abc"`, 1), + "a bad date": strings.Replace(v2(""), `"date":"2026-10-26"`, `"date":"26.10.2026"`, 1), + "a bad time": strings.Replace(v2(""), `"at":"2026-10-26T06:00:00Z"`, `"at":"yesterday"`, 1), + "a short report": strings.Replace(v2(""), strings.Repeat("b", 64), "bbbb", 1), + "a closing on another line": strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"backport-batch/release-1.0-20261026","head":"`+sha+`"}`, 1), + "a closing without a head": strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"`+batchBranch1+`","head":""}`, 1), + "a negative stale": strings.Replace(v2(""), `"stale":[9]`, `"stale":[-9]`, 1), + "an unknown key": v2(`,"parked":[]`), + "an entry of no kind": strings.Replace(v2(""), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"files":[]}`, 1), + "another line": strings.Replace(v2(""), `"line":"release-1.1"`, `"line":"release-1.0"`, 1), + "an escape in a reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"fails\u001b[2J the e2e"`, 1), + "a newline in a reason": strings.Replace(v2(""), `"reason":"fails the e2e"`, `"reason":"fails\nwarning: forged"`, 1), + "an escape in a drop": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"needs\u001b]0;x\u0007 #4"`, 1), + "a newline in a drop": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"needs #4\nnote: forged"`, 1), + "a reason past its length": strings.Replace(v2(""), `"reason":"needs #4"`, `"reason":"`+strings.Repeat("x", titleLimit+1)+`"`, 1), + "a bidi file name": strings.Replace(v2(""), `"files":["app/five.txt"]`, `"files":["app/\u202efive.txt"]`, 1), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } + closing := strings.Replace(v2(""), `"closing":null`, `"closing":{"pr":90,"branch":"`+batchBranch1+`","head":"`+sha+`"}`, 1) + if rec, err := parseRecord(block(closing), l); err != nil || rec.Closing == nil || rec.Closing.PR != 90 { + t.Errorf("a record that is closing = %+v, %v", rec, err) + } +} + +// recordPR is a batch pull request on release-1.1 carrying body, numbered n, +// in state, opened by login. +func recordPR(n int, state, login, body string) pullRequest { + pr := notifyPR("20261026", body) + pr.Number, pr.State = n, state + pr.Head.Ref = fmt.Sprintf("backport-batch/release-1.1-202610%02d", n%30+1) + pr.User.Login = login + if strings.HasSuffix(login, "[bot]") { + pr.User.Type = "Bot" + } + return pr +} + +func recordBody(conflict int) string { + return "text\n\n" + deferredFence + "\n" + fmt.Sprintf(`{"line":"release-1.1","deferred":[{"original":%d,"commit":"%s","files":["a"]}]}`, conflict, strings.Repeat("a", 40)) + "\n```\n" +} + +// The record is the block of the newest batch pull request that carries one, +// whatever its state, among those a trusted author opened from a batch branch +// of this repository; anything that leaves the newest record in doubt stops +// the command rather than falling back to an older one. +func TestFindLineRecord(t *testing.T) { + const bot = defaultBatchBot + other := func(n int) pullRequest { + pr := recordPR(n, "closed", "maintainer", "") + pr.Head.Ref = fmt.Sprintf("fix-%d", n) + return pr + } + fork := recordPR(92, "closed", "maintainer", recordBody(92)) + fork.Head.Repo = &struct { + FullName string `json:"full_name"` + }{"someone/cozystack"} + cases := []struct { + name string + prs []pullRequest + open int // the open batch the caller found, or 0 + bot string + carrier int // 0 for none + isOpen bool + refusal string + pages int + }{ + {name: "the open batch first", prs: []pullRequest{recordPR(92, "open", "maintainer", recordBody(92)), recordPR(91, "closed", "maintainer", recordBody(91))}, open: 92, carrier: 92, isOpen: true}, + {name: "the newest closed record", prs: []pullRequest{recordPR(91, "closed", bot, recordBody(91)), recordPR(90, "closed", bot, recordBody(90))}, bot: bot, carrier: 91}, + {name: "a person's batch without a block is passed over", prs: []pullRequest{recordPR(92, "closed", "maintainer", "picked by hand"), recordPR(91, "closed", "maintainer", recordBody(91))}, carrier: 91}, + {name: "the bot's batch without a block", prs: []pullRequest{recordPR(92, "closed", bot, "edited away"), recordPR(91, "closed", bot, recordBody(91))}, bot: bot, refusal: "carries no ```backport-batch block"}, + {name: "a fork is no batch", prs: []pullRequest{fork, recordPR(91, "closed", "maintainer", recordBody(91))}, carrier: 91}, + {name: "a newer record by someone untrusted", prs: []pullRequest{recordPR(92, "closed", "someone", recordBody(92)), recordPR(91, "closed", "maintainer", recordBody(91))}, refusal: "#92"}, + {name: "a closed record newer than the open batch", prs: []pullRequest{recordPR(92, "closed", "maintainer", recordBody(92)), recordPR(91, "open", "maintainer", recordBody(91))}, open: 91, refusal: "newer than the open batch pull request #91"}, + {name: "a record on the second page", prs: append(func() (out []pullRequest) { + for n := 1000; n < 1000+recentPullsPage; n++ { + out = append(out, other(n)) + } + return out + }(), recordPR(91, "closed", "maintainer", recordBody(91))), carrier: 91, pages: 2}, + {name: "ten full pages without a record", prs: func() (out []pullRequest) { + for n := 1000; n < 1000+10*recentPullsPage+1; n++ { + out = append(out, other(n)) + } + return out + }(), refusal: "pull requests", pages: 10}, + {name: "the bot's record when no bot is trusted", prs: []pullRequest{recordPR(91, "closed", bot, recordBody(91))}, refusal: "#91"}, + {name: "a hand-written open batch beside a closed record", prs: []pullRequest{recordPR(92, "open", "maintainer", "picked by hand"), recordPR(91, "closed", "maintainer", recordBody(91))}, open: 92, carrier: 91}, + {name: "no batch ever", prs: []pullRequest{other(5)}}, + {name: "an unreadable record", prs: []pullRequest{recordPR(91, "closed", "maintainer", "text\n\n"+deferredFence+"\n{\n```\n")}, refusal: "#91 (closed)"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + l, _ := parseLine("release-1.1") + gh := &fakeGH{prs: tc.prs, bot: tc.bot} + var open *pullRequest + for i := range tc.prs { + if tc.prs[i].Number == tc.open { + open = &tc.prs[i] + } + } + src, err := findLineRecord(gh, l, open) + switch { + case tc.refusal != "": + if err == nil || !strings.Contains(err.Error(), tc.refusal) { + t.Errorf("findLineRecord = %+v, %v; want a refusal with %q", src, err, tc.refusal) + } + case err != nil: + t.Errorf("findLineRecord: %v", err) + case tc.carrier == 0 && (src.carrier != nil || src.rec != nil): + t.Errorf("found #%d, want no record", src.carrier.Number) + case tc.carrier != 0 && (src.carrier == nil || src.carrier.Number != tc.carrier || src.rec == nil || src.open != tc.isOpen || src.rec.Deferred[0].Original != tc.carrier): + t.Errorf("found %+v, want #%d open=%v", src, tc.carrier, tc.isOpen) + } + if tc.pages != 0 && gh.listed != tc.pages { + t.Errorf("read %d pages, want %d", gh.listed, tc.pages) + } + if tc.open != 0 && src.code != open { + t.Errorf("the batch to build on is %+v, want #%d", src.code, tc.open) + } + }) + } +} + +// A run that picks nothing while no batch is open writes nothing, since +// merged and closed batches are never written: it says its classification is +// not kept, and check still reads the merged batch's record. +func TestZeroPickRunLeavesNoRecord(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n", "app/seven.txt": "line\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mergeBatch(f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + edits, comments := len(gh.edited), len(gh.issueComments) + + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 0 || !strings.Contains(stderr, "no batch pull request is open on release-1.1; this run's classification is not retained") { + t.Errorf("run = %d, want 0 saying nothing is retained:\n%s", code, stderr) + } + if len(gh.created) != 1 || len(gh.edited) != edits || len(gh.issueComments) != comments || f.remoteHead(batchBranch1) != "" { + t.Errorf("created %q, edited %q, %d comments, branch at %q; want nothing written", gh.created, gh.edited, len(gh.issueComments), f.remoteHead(batchBranch1)) + } + f.git("switch", "-q", "main") + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 1 || !strings.Contains(stdout, "/pull/7 ") || strings.Contains(stdout, "/pull/5 ") || !strings.Contains(stderr, "deferred conflict on #90 (merged)") { + t.Errorf("check = %d, want 1 for #7 alone and #5 still a deferred conflict:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } +} + +// The open batch's author is asked for once although both the open batch and +// its record are read. +func TestRecordAsksForAPermissionOnce(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + gh.asked = nil + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 { + t.Fatalf("check = %d\n%s", code, stderr) + } + if len(gh.asked) != 1 { + t.Errorf("asked GitHub for %q, want the author once", gh.asked) + } +} + +// The record and the report name what the classification was made of: the +// heads it read, the policy, the tool, the job, the time and the flags. +func TestRecordNamesItsInputs(t *testing.T) { + f := newFixture(t) + f.lines() + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.onLine("release-1.1", "fix(app): five\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "5\n"}) + f.tool = strings.Repeat("c", 40) + "+dirty" + t.Setenv("GITHUB_SERVER_URL", "https://github.com") + t.Setenv("GITHUB_REPOSITORY", "cozystack/cozystack") + t.Setenv("GITHUB_RUN_ID", "123") + t.Setenv("GITHUB_RUN_ATTEMPT", "2") + gh := &fakeGH{} + stdout := mustRun(t, f, gh, "run", "--line", "release-1.0", "--no-workflow-files", "--push") + l, _ := parseLine("release-1.0") + const branch = "backport-batch/release-1.0-20261026" + rec, err := parseRecord(stdout, l) + if err != nil { + t.Fatal(err) + } + want := builtFrom{Main: f.git("rev-parse", "origin/main"), Line: f.git("rev-parse", "origin/release-1.0"), Newer: f.git("rev-parse", "origin/release-1.1"), + Policy: f.git("rev-parse", "origin/main"), Batch: f.remoteHead(branch), Tool: f.tool, Run: "https://github.com/cozystack/cozystack/actions/runs/123/attempts/2", + At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: true, Mode: modeNew}} + if rec.Version != 2 || *rec.Built != want || bodyReportOf(t, stdout, l).doc.Built != want || rec.Report != bodyReportOf(t, stdout, l).doc.ID { + t.Errorf("record = %+v, built %+v; want built %+v and the report it names", rec, rec.Built, want) + } + + // A policy from another ref, a run by hand that continues the batch. + f.git("push", "-q", "origin", "origin/main:refs/heads/policy-draft") + f.git("fetch", "-q", "origin") + t.Setenv("GITHUB_RUN_ID", "") + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + mc6 := f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.onLine("release-1.1", "fix(app): six\n\n(cherry picked from commit "+mc6+")", map[string]string{"app/six.txt": "6\n"}) + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--continue", "--policy", "origin/policy-draft", "--push") + if rec, err = parseRecord(stdout, l); err != nil { + t.Fatal(err) + } + if b := rec.Built; b.Policy != f.git("rev-parse", "origin/policy-draft") || b.Main != f.git("rev-parse", "origin/main") || b.Run != "" || b.Options != (runOptions{Mode: modeContinue}) || b.Batch != f.remoteHead(branch) { + t.Errorf("built %+v; want the draft's policy, no job and a continuation without the flag", b) + } +} + +// What a run assembled with --no-workflow-files stays saved through the +// operations that only change the record, which take no such flag: check +// keeps failing the cut on the wait they leave in place. +func TestMetadataOperationsKeepTheMode(t *testing.T) { + for _, v1 := range []bool{false, true} { + t.Run(fmt.Sprintf("an open record of version 1: %v", v1), func(t *testing.T) { + f := workflowWorld(t) + f.onLine("release-1.1", "fix(app): a line-only nine", map[string]string{"app/nine.txt": "line\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + f.pr(9, "fix(app): nine", "", map[string]string{"app/nine.txt": "9\n"}) + gh := &fakeGH{} + stdout := mustRun(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + reopen(f, gh) + if v1 { + waits := []pick{{v: &verdict{o: &original{number: 5}}, files: []string{".github/workflows/five.yaml"}}} + gh.prs[0].Body = strings.Replace(stdout, strings.Split(strings.Split(stdout, deferredFence+"\n")[1], "\n")[0], strings.Split(deferredBlock(notifyLine, nil, nil, waits), "\n")[1], 1) + } else if rec, err := parseRecord(stdout, notifyLine); err != nil || !rec.Built.Options.NoWorkflowFiles { + t.Fatalf("the scheduled run saved %+v, %v", rec, err) + } + for _, op := range []struct { + args []string + mode string + }{ + {[]string{"--drop", "9", "--reason", "needs #4"}, modeDrop}, + {[]string{"--undrop", "9"}, modeUndrop}, + {[]string{"--defer", "7", "--reason", "fails the e2e"}, modeDefer}, + } { + stdout = mustRun(t, f, gh, append(append([]string{"run", "--line", "release-1.1"}, op.args...), "--push")...) + rec, err := parseRecord(stdout, notifyLine) + if err != nil || rec.Built.Options != (runOptions{NoWorkflowFiles: true, Mode: op.mode}) { + t.Errorf("after %q the record is %+v, %v; want the flag kept", op.args, rec.Built, err) + } + reopen(f, gh) + } + var rec *lineRecord + var err error + reopen(f, gh) + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/") { + t.Errorf("check --cut = %d, want #5 still waiting:\n%s", code, stdout) + } + + // Deferring the last picks closes the batch, and its record + // still says how the batch was assembled. + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "6,8", "--reason", "fails the e2e", "--push") + rec, err = parseRecord(gh.find(90).Body, notifyLine) + if err != nil || gh.find(90).State != "closed" || rec.Closing == nil || rec.Built.Options != (runOptions{NoWorkflowFiles: true, Mode: modeClose}) { + t.Errorf("after the close the record is %+v, %v; want the flag kept", rec.Built, err) + } + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check --cut = %d, want #5 missing once no batch lists it:\n%s", code, stdout) + } + }) + } +} + +// A state is named only when it is one GitHub uses; anything else is quoted. +func TestPRStateNamesOnlyKnownStates(t *testing.T) { + at := "2026-10-26T12:00:00Z" + for _, tc := range []struct { + pr pullRequest + want string + }{ + {pullRequest{State: "open"}, "open"}, + {pullRequest{State: ""}, "open"}, + {pullRequest{State: "closed"}, "closed"}, + {pullRequest{State: "closed", MergedAt: &at}, "merged"}, + {pullRequest{State: "open\n\x1b[2Jforged"}, `in state "open\n\x1b[2Jforged"`}, + } { + if got := prState(&tc.pr); got != tc.want { + t.Errorf("prState(%q) = %q, want %q", tc.pr.State, got, tc.want) + } + } +} diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go new file mode 100644 index 000000000..46e37c955 --- /dev/null +++ b/cmd/backport-batch/report.go @@ -0,0 +1,752 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "cmp" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "regexp" + "runtime/debug" + "slices" + "strconv" + "strings" + "time" +) + +// A report is one classification of a line: where every original merged into +// main since the line branched stands, and what it was made of. The batch +// pull request's body carries the latest one beside the record, and an issue +// comment the tool never edits keeps each one, so a later run that replaces +// the body loses none of them. Its ID hashes the classification, not the pull +// request it lands on, which a new batch does not have yet when the report is +// made. + +const ( + reportFence = "```backport-batch-report" + reportMarker = "\n" + + "
Backport batch report for " + s.doc.Line + ", " + date + " (" + s.doc.ID[:12] + ")\n\n" + + reportBlock(s.raw) + "\n
\n" + if len(c) > reportLimit { + return "", fmt.Errorf("the report comment is %d bytes, over the %d it may have", len(c), reportLimit) + } + return c, nil +} + +var reportMarkerRE = regexp.MustCompile(`^` + regexp.QuoteMeta(reportMarker) + `([0-9a-f]{64}) -->\n`) + +// findReports reads the report comments on batch pull request n, against l, +// by authors the batch trusts, as it trusts a record's author, by ID. One +// whose content does not match the ID its marker gives, or is not what a run +// writes, is returned apart, with why; a look-alike by anyone else is not +// read, whatever it says. +func findReports(gh github, n int, l line) (map[string]storedReport, map[string]string, error) { + comments, err := gh.comments(n) + if err != nil { + return nil, nil, err + } + found, bad := map[string]storedReport{}, map[string]string{} + for _, c := range comments { + if !strings.HasPrefix(c.Body, reportMarker) { + continue + } + why, err := trustedAuthor(gh, c.User.Login, c.User.Type) + if err != nil { + return nil, nil, err + } + if why != "" { + continue + } + m := reportMarkerRE.FindStringSubmatch(c.Body) + if m == nil { + bad[strconv.FormatInt(c.ID, 10)] = fmt.Sprintf("comment %d starts like a report but names none", c.ID) + continue + } + blocks, err := fencedLines(c.Body, reportFence) + if err == nil && len(blocks) != 1 { + err = fmt.Errorf("it holds %d reports", len(blocks)) + } + var doc reportDoc + if err == nil { + doc, err = parseReport(blocks[0], l) + } + if err == nil && doc.ID != m[1] { + err = fmt.Errorf("its marker names %.12s", m[1]) + } + if err != nil { + bad[m[1]] = fmt.Sprintf("comment %d: %v", c.ID, err) + continue + } + found[m[1]] = storedReport{doc: doc, raw: blocks[0]} + } + return found, bad, nil +} + +// toolVersion is the commit this binary was built from, with +dirty when its +// tree had changes, or "" when the build did not record it, as go run does +// not. +func toolVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + rev, dirty := "", false + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + rev = s.Value + case "vcs.modified": + dirty = s.Value == "true" + } + } + if checkSHA(rev) != nil { + return "" + } + if dirty { + rev += "+dirty" + } + return rev +} + +// runURL is the Actions run the command runs in, from the variables GitHub +// sets for it, or "" outside one. +func runURL(lookup func(string) (string, bool)) string { + get := func(key string) string { v, _ := lookup(key); return v } + url := get("GITHUB_SERVER_URL") + "/" + get("GITHUB_REPOSITORY") + "/actions/runs/" + get("GITHUB_RUN_ID") + if attempt := get("GITHUB_RUN_ATTEMPT"); attempt != "" && attempt != "1" { + url += "/attempts/" + attempt + } + if !runURLRE.MatchString(url) { + return "" + } + return url +} + +// report prints a report a batch pull request keeps, writing nothing, so it +// works on merged and closed batches too. It reads GitHub, and for a report +// that keeps drops also this clone's history at the heads the report was +// made from, which it never fetches. +func (a *app) report(args []string) int { + fs := a.flags("report") + number := fs.Int("pr", 0, "the batch pull request") + id := fs.String("id", "", "the report, by its ID (default: the one the body carries)") + asJSON := fs.Bool("json", false, "print the report's JSON") + if err := parse(fs, args); err != nil { + return 2 + } + if *number <= 0 { + return a.fail(fmt.Errorf("--pr %d is not a pull request number", *number)) + } + if *id != "" && !reportIDRE.MatchString(*id) { + return a.fail(fmt.Errorf("--id %q is not a report ID", *id)) + } + pr, err := a.gh.pull(*number) + if err != nil { + return a.fail(err) + } + l, err := parseLine(pr.Base.Ref) + if m := batchBranchRE.FindStringSubmatch(pr.Head.Ref); err != nil || m == nil || m[1] != l.branch() || pr.Head.Repo == nil || pr.Head.Repo.FullName != pr.Base.Repo.FullName { + return a.fail(fmt.Errorf("#%d is not a batch pull request of this repository", pr.Number)) + } + why, err := trusted(a.gh, pr) + if err != nil { + return a.fail(err) + } + if why != "" { + return a.fail(fmt.Errorf("batch pull request #%d was %s, so what it says is not the batch's", pr.Number, why)) + } + inBody, err := bodyReport(pr.Body, l) + if err != nil { + return a.fail(fmt.Errorf("batch pull request #%d: %w", pr.Number, err)) + } + if *id == "" { + if inBody == nil { + return a.fail(fmt.Errorf("batch pull request #%d carries no report in its body; name one with --id", pr.Number)) + } + *id = inBody.doc.ID + } + posted, bad, err := findReports(a.gh, pr.Number, l) + if err != nil { + return a.fail(err) + } + if why := bad[*id]; why != "" { + return a.fail(fmt.Errorf("batch pull request #%d: the report %.12s is not what its comment says: %s", pr.Number, *id, why)) + } + s, inComment := posted[*id] + switch { + case inComment: + case inBody != nil && inBody.doc.ID == *id: + s = *inBody + default: + return a.fail(fmt.Errorf("batch pull request #%d keeps no report %.12s", pr.Number, *id)) + } + if *asJSON { + fmt.Fprintln(a.stdout, s.raw) + return 0 + } + a.printReport(&pr, s, inBody, inComment) + return 0 +} + +// printReport prints a report. It was checked when it was read, and every +// field still goes through terminalSafe, so nothing it carries spans lines +// or drives a terminal. +func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, inComment bool) { + d, b := s.doc, s.doc.Built + safe := terminalSafe + fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode)) + if b.Options.NoWorkflowFiles { + fmt.Fprint(a.stdout, " with --no-workflow-files") + } + fmt.Fprintln(a.stdout) + fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none"))) + current := "none" + if inBody != nil { + current = safe(inBody.doc.ID) + } + switch { + case inBody != nil && inBody.doc.ID == d.ID: + fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number) + case pr.Head.SHA == b.Batch: + fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current) + default: + fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current) + } + if !inComment { + if prState(pr) == "open" { + fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it") + } else { + fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr)) + } + } + type line struct { + n int + text string + } + var lines []line + for _, n := range d.Primary.Picked { + lines = append(lines, line{n, "picked"}) + } + for _, n := range d.Primary.Missing { + lines = append(lines, line{n, "missing"}) + } + for _, e := range d.Primary.Deferred { + text := "deferred: " + switch { + case e.Workflows: + text += "waits for a maintainer, changes " + strings.Join(e.Files, ", ") + case e.Reason != "": + text += e.Reason + default: + text += fmt.Sprintf("conflict at %.12s in %s", e.Commit, strings.Join(e.Files, ", ")) + } + if e.More > 0 { + text += fmt.Sprintf(" and %d more", e.More) + } + lines = append(lines, line{e.Original, text}) + } + for _, l := range d.Primary.Skipped.lists() { + for _, n := range *l.list { + lines = append(lines, line{n, "skipped." + l.name}) + } + } + slices.SortFunc(lines, func(x, y line) int { return cmp.Compare(x.n, y.n) }) + for _, l := range lines { + fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text)) + } + overlay := func(name string, list []int) { + if len(list) > 0 { + fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]")) + } + } + overlay("stale, still on the batch", d.Overlays.Stale) + for _, e := range d.Overlays.Dropped { + fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason)) + } + overlay("changed in the pick", d.Overlays.Changed) + if len(d.Other) > 0 { + other := make([]string, len(d.Other)) + for i, sha := range d.Other { + other[i] = safe(sha) + } + fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", ")) + } +} + +func or(s, otherwise string) string { + if s == "" { + return otherwise + } + return s +} + +// postReport writes a report comment on pull request n, after GitHub says +// it is still open: nothing is written to a merged or closed one. +func (a *app) postReport(n int, s storedReport) error { + if err := a.stillOpen(n); err != nil { + return err + } + c, err := reportComment(s) + if err != nil { + return err + } + name, err := writeTemp("backport-batch-report-*.md", c) + if err != nil { + return err + } + defer os.Remove(name) + return a.gh.createComment(n, name) +} + +// stillOpen reads pull request n again and refuses to go on unless it is +// open. +func (a *app) stillOpen(n int) error { + pr, err := a.gh.pull(n) + if err != nil { + return err + } + if st := prState(&pr); st != "open" { + return fmt.Errorf("batch pull request #%d is %s now; nothing is written to it", n, st) + } + return nil +} + +func writeTemp(pattern, content string) (string, error) { + f, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + if _, err := f.WriteString(content); err != nil { + f.Close() + os.Remove(f.Name()) + return "", err + } + if err := f.Close(); err != nil { + os.Remove(f.Name()) + return "", err + } + return f.Name(), nil +} diff --git a/cmd/backport-batch/report_test.go b/cmd/backport-batch/report_test.go new file mode 100644 index 000000000..b11208a04 --- /dev/null +++ b/cmd/backport-batch/report_test.go @@ -0,0 +1,707 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + "unicode" +) + +// reportsPosted lists the report IDs the comments carry, in the order they +// were posted, whoever wrote them. +func reportsPosted(gh *fakeGH) []string { + var ids []string + for _, c := range gh.issueComments { + if m := reportMarkerRE.FindStringSubmatch(c.Body); m != nil { + ids = append(ids, m[1]) + } + } + return ids +} + +// bodyReportOf is the report a body on release-1.1, or on the line given, +// carries, failing the test without one. +func bodyReportOf(t *testing.T, body string, on ...line) storedReport { + t.Helper() + l := notifyLine + if len(on) > 0 { + l = on[0] + } + s, err := bodyReport(body, l) + if err != nil || s == nil { + t.Fatalf("the body carries no readable report: %v\n%s", err, body) + } + return *s +} + +// refusePushes makes the remote refuse every push, as a lost connection or +// a protected branch would, until it is called with false. +func (f *fixture) refusePushes(on bool) { + f.t.Helper() + hook := filepath.Join(f.remote, "hooks", "pre-receive") + if !on { + if err := os.Remove(hook); err != nil { + f.t.Fatal(err) + } + return + } + if err := os.WriteFile(hook, []byte("#!/bin/sh\necho refused >&2\nexit 1\n"), 0o755); err != nil { + f.t.Fatal(err) + } +} + +func mustRun(t *testing.T, f *fixture, gh *fakeGH, args ...string) string { + t.Helper() + code, stdout, stderr := f.exec(gh, args...) + if code != 0 { + t.Fatalf("%q = %d\n%s", args, code, stderr) + } + f.git("switch", "-q", "main") + return stdout +} + +// A report comment keeps each classification: one the next run replaced in +// the body is still read back by its ID, also once the batch merged, which +// nothing writes to. +func TestReportSurvivesContinuation(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only five", map[string]string{"app/five.txt": "line\n"}) + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + r1 := bodyReportOf(t, gh.prs[0].Body).doc.ID + f.onLine("release-1.1", "fix(app): five, by hand\n\n(cherry picked from commit "+mc5+")", map[string]string{"app/five.txt": "5\n"}) + f.now = f.now.AddDate(0, 0, 1) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + r2 := bodyReportOf(t, gh.find(90).Body).doc.ID + if r1 == r2 || !slices.Equal(reportsPosted(gh), []string{r1, r2}) { + t.Fatalf("reports %q; want R1 %.12s, then R2 %.12s", reportsPosted(gh), r1, r2) + } + read := func(id, want string) { + t.Helper() + code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", id) + if code != 0 || !strings.Contains(stdout, want) { + t.Errorf("report --id %.12s = %d, want %q:\n%s%s", id, code, want, stdout, stderr) + } + } + for _, merged := range []bool{false, true} { + if merged { + mergeBatch(f, gh) + } + read(r1, "#5 deferred: conflict at ") + read(r2, "#5 skipped.on-line") + read(r2, "#6 picked") + } +} + +// Two runs that find the same post one report, and the body carries the one +// kept, whatever time and job the second ran in. A look-alike someone else +// wrote is no report, and a kept report edited since stops the reader. +func TestReportDedupAndTrust(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + t.Setenv("GITHUB_SERVER_URL", "https://github.com") + t.Setenv("GITHUB_REPOSITORY", "cozystack/cozystack") + t.Setenv("GITHUB_RUN_ID", "100") + t.Setenv("GITHUB_RUN_ATTEMPT", "1") + + // The report the next run makes, read off a run that publishes nothing, + // planted by someone the batch does not trust. + preview := bodyReportOf(t, mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue")) + planted, err := reportComment(preview) + if err != nil { + t.Fatal(err) + } + look := issueComment{ID: 7, Body: planted} + look.User.Login, look.User.Type = "someone", "User" + gh.issueComments = append(gh.issueComments, look) + + f.now = f.now.AddDate(0, 0, 1) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + kept := bodyReportOf(t, gh.find(90).Body) + if kept.doc.ID != preview.doc.ID { + t.Fatalf("the run made %.12s, the preview %.12s", kept.doc.ID, preview.doc.ID) + } + f.now = f.now.AddDate(0, 0, 1) + t.Setenv("GITHUB_RUN_ID", "101") + mustRun(t, f, gh, "run", "--line", "release-1.1", "--continue", "--push") + posted := reportsPosted(gh) + if n := len(slices.DeleteFunc(slices.Clone(posted), func(id string) bool { return id != kept.doc.ID })); n != 2 || len(posted) != 3 { + t.Errorf("reports %q; want the first, the look-alike and the run's own once", posted) + } + body := gh.find(90).Body + again := bodyReportOf(t, body) + rec, err := parseRecord(body, notifyLine) + if err != nil || again.raw != kept.raw || rec.Report != again.doc.ID || !strings.Contains(again.raw, `"run":"https://github.com/cozystack/cozystack/actions/runs/100"`) { + t.Errorf("the second run's body carries %s with record report %v (%v); want the kept bytes of run 100", again.raw, rec, err) + } + + // The reader takes the run's own comment, not the look-alike. + if code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", kept.doc.ID); code != 0 || !strings.Contains(stdout, "#6 picked") { + t.Errorf("report = %d:\n%s%s", code, stdout, stderr) + } + for i := range gh.issueComments { + c := &gh.issueComments[i] + if c.User.Login == "maintainer" && strings.Contains(c.Body, kept.doc.ID) { + c.Body = strings.Replace(c.Body, `"picked":[5,6]`, `"picked":[5]`, 1) + } + } + if code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", kept.doc.ID); code != 2 || !strings.Contains(stderr, "does not match") { + t.Errorf("report of an edited comment = %d, want 2:\n%s%s", code, stdout, stderr) + } +} + +// The report holds no pull request number, so a new batch's report is whole +// before GitHub numbers the pull request, and the reader finds it on +// whatever number it got. +func TestFirstReportIDIsKnownBeforeTheNumber(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{nextPR: 4242} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + inBody := bodyReportOf(t, gh.bodies[0]) + // Every list is one, empty or not, so equal reports are equal bytes: + // main here has no commit that merged no pull request. + if !strings.Contains(inBody.raw, `"other":[]`) { + t.Errorf("the report writes its empty list of other commits as something else:\n%s", inBody.raw) + } + if !slices.Equal(gh.commented, []int{4242}) || !slices.Equal(reportsPosted(gh), []string{inBody.doc.ID}) { + t.Fatalf("commented on %v with %q, want #4242 with %.12s", gh.commented, reportsPosted(gh), inBody.doc.ID) + } + found, bad, err := findReports(gh, 4242, notifyLine) + if err != nil || len(bad) != 0 || found[inBody.doc.ID].raw != inBody.raw { + t.Errorf("findReports = %v, %v, %v; want the body's report", found, bad, err) + } + if code, stdout, stderr := f.exec(gh, "report", "--pr", "4242"); code != 0 || !strings.Contains(stdout, "report "+inBody.doc.ID) || !strings.Contains(stdout, "#4242's body carries this report") { + t.Errorf("report --pr 4242 = %d:\n%s%s", code, stdout, stderr) + } +} + +// Every original main merged since the line branched is in exactly one +// category of the report, a commit that merged no pull request is counted +// apart, and the report says so whatever the run had to do. +func TestReportPartition(t *testing.T) { + f := newFixture(t) + f.lines() + gh := &fakeGH{labels: map[string][]int{labelSkip: {9}}} + // onNewer puts an original on release-1.1, as the newer line's batch + // would, so rule 6 lets it onto release-1.0. + onNewer := func(mc string, files map[string]string) { + f.onLine("release-1.1", "fix: by the newer batch\n\n(cherry picked from commit "+mc+")", files) + } + pr := func(n int, title string, files map[string]string, newer bool) { + mc := f.pr(n, title, "", files) + if newer { + onNewer(mc, files) + } + } + t.Setenv("GIT_COMMITTER_DATE", "2026-01-01T00:00:00Z") + f.pr(4, "fix(app): four, before the start", "", map[string]string{"app/four.txt": "4\n"}) + os.Unsetenv("GIT_COMMITTER_DATE") + f.setPolicy("release-1.0: 2026-06-01\n") + f.onLine("release-1.0", "fix(app): a line-only eleven", map[string]string{"app/eleven.txt": "line\n"}) + pr(5, "fix(app): five", map[string]string{"app/five.txt": "5\n"}, true) + pr(6, "fix(app): six, not on the newer line", map[string]string{"app/six.txt": "6\n"}, false) + pr(7, "fix(app)!: seven", map[string]string{"app/seven.txt": "7\n"}, true) + pr(8, "fix(platform): eight", map[string]string{migrationsDir + "42": "#!/bin/sh\n"}, true) + pr(9, "fix(app): nine", map[string]string{"app/nine.txt": "9\n"}, true) + mc10 := f.pr(10, "fix(app): ten", "", map[string]string{"app/ten.txt": "10\n"}) + onNewer(mc10, map[string]string{"app/ten.txt": "10\n"}) + f.onLine("release-1.0", "fix(app): ten\n\n(cherry picked from commit "+mc10+")", map[string]string{"app/ten.txt": "10\n"}) + pr(11, "fix(app): eleven", map[string]string{"app/eleven.txt": "11\n"}, true) + pr(12, "fix(ci): twelve", map[string]string{".github/workflows/twelve.yaml": "on: push\n"}, true) + pr(13, "fix(app): thirteen", map[string]string{"app/thirteen.txt": "13\n"}, true) + f.git("switch", "-q", "main") + f.write("app/direct.txt", "pushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: a direct push") + direct := f.git("rev-parse", "HEAD") + f.git("push", "-q", "origin", "main") + + stdout := mustRun(t, f, gh, "run", "--line", "release-1.0", "--no-workflow-files", "--push") + const branch = "backport-batch/release-1.0-20261026" + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + pr(14, "fix(app): fourteen, after the batch", map[string]string{"app/fourteen.txt": "14\n"}, true) + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--defer", "13", "--reason", "fails the e2e", "--push") + // An active drop, of #5, which the batch carries, and one of a feat, + // which rule 2 keeps out first. + gh.prs = []pullRequest{f.batchPR(90, "release-1.0", branch, stdout)} + stdout = mustRun(t, f, gh, "run", "--line", "release-1.0", "--drop", "5,1", "--reason", "not for 1.0", "--push") + + l, _ := parseLine("release-1.0") + s := bodyReportOf(t, stdout, l) + got, err := s.doc.categories() + if err != nil { + t.Fatal(err) + } + want := map[int]string{1: "skipped.type", 4: "skipped.before-start", 5: "skipped.dropped", 6: "skipped.newer", 7: "skipped.breaking", + 8: "skipped.touches", 9: "skipped.skip-label", 10: "skipped.on-line", 11: "deferred", 12: "deferred", 13: "deferred", + 14: "missing", 9000: "skipped.type"} + for n, category := range want { + if got[n] != category { + t.Errorf("#%d is %q, want %q", n, got[n], category) + } + delete(got, n) + } + if len(got) != 0 { + t.Errorf("the report also names %v", got) + } + if gh.find(90).State != "closed" || !strings.Contains(stdout, `"mode":"close"`) { + t.Errorf("dropping the batch's last pick left #90 %s; want it closed", gh.find(90).State) + } + if !slices.Equal(droppedNumbers(s.doc.Overlays.Dropped), []int{1, 5}) || len(s.doc.Primary.Picked) != 0 { + t.Errorf("the report drops %v and picks %v; want #1 and #5 dropped, nothing picked", s.doc.Overlays.Dropped, s.doc.Primary.Picked) + } + if !slices.Contains(s.doc.Other, direct) { + t.Errorf("the report does not count the direct push %.12s: %v", direct, s.doc.Other) + } + kinds := map[int]string{} + for _, e := range s.doc.Primary.Deferred { + kinds[e.Original] = fmt.Sprintf("commit=%v reason=%q workflows=%v", e.Commit != "", e.Reason, e.Workflows) + } + if kinds[11] != `commit=true reason="" workflows=false` || kinds[12] != `commit=false reason="" workflows=true` || kinds[13] != `commit=false reason="fails the e2e" workflows=false` { + t.Errorf("the deferred entries are %v", kinds) + } +} + +// A report too big for a comment is refused before anything is written: no +// push, no comment, no edit, and no replay of the earlier report either. +func TestReportTooLargeIsRefusedBeforePush(t *testing.T) { + f, gh, head := bigWaits(t, 7) + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push") + if code != 2 || !strings.Contains(stderr, "the report comment is") { + t.Errorf("run --push = %d, want a refusal of the report:\n%s", code, stderr) + } + if f.remoteHead(batchBranch1) != head || len(gh.issueComments) != 0 || len(gh.edited) != 0 { + t.Errorf("the refused run moved the batch to %.12s, wrote %d comments and edited %q", f.remoteHead(batchBranch1), len(gh.issueComments), gh.edited) + } +} + +// bigWaits opens a batch whose report comment failed, so its body keeps a +// report no comment does, and then merges n originals that each wait on 50 +// workflow files with long names, which both the record and the report list +// in full. +func bigWaits(t *testing.T, n int) (*fixture, *fakeGH, string) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{fail: map[string]error{"createComment": errString("HTTP 502")}} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 2 || len(gh.created) != 1 { + t.Fatalf("the first run = %d, want its comment to fail after the pull request opened\n%s", code, stderr) + } + f.git("switch", "-q", "main") + for i := range n { + files := map[string]string{} + for j := range filesKept { + files[fmt.Sprintf(".github/workflows/%s-%02d-%02d.yaml", strings.Repeat("w", 170), i, j)] = "on: push\n" + } + f.pr(10+i, fmt.Sprintf("fix(ci): wait %d", i), "", files) + } + f.now = f.now.AddDate(0, 0, 1) + return f, gh, f.remoteHead(batchBranch1) +} + +// A failure between any two writes leaves what was written readable, and the +// next run, whatever changed meanwhile, replays a report the body kept but no +// comment did, byte for byte, and posts nothing twice. A batch closed in +// between is never written again. +func TestReportReplayAfterEachFailure(t *testing.T) { + setup := func(t *testing.T) (*fixture, *fakeGH) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f, &fakeGH{} + } + // change moves a label, main, the clock and the run's flags between + // the failure and the retry, so the retry's report is another one. + change := func(f *fixture, gh *fakeGH) []string { + gh.labels = map[string][]int{labelSkip: {6}} + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.now = f.now.AddDate(0, 0, 1) + return []string{"run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push"} + } + once := func(t *testing.T, gh *fakeGH, ids ...string) { + t.Helper() + posted := reportsPosted(gh) + for _, id := range ids { + if n := len(slices.DeleteFunc(slices.Clone(posted), func(p string) bool { return p != id })); n != 1 { + t.Errorf("report %.12s is posted %d times: %q", id, n, posted) + } + } + } + + t.Run("a new batch whose pull request was not opened", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if len(gh.prs) != 0 || len(gh.issueComments) != 0 || f.remoteHead(batchBranch1) == "" { + t.Errorf("after the failure: %d pull requests, %d comments, branch %q; want the branch alone", len(gh.prs), len(gh.issueComments), f.remoteHead(batchBranch1)) + } + mustRun(t, f, gh, change(f, gh)...) + if len(gh.prs) != 1 || len(reportsPosted(gh)) != 1 || bodyReportOf(t, gh.prs[0].Body).doc.ID != reportsPosted(gh)[0] { + t.Errorf("the retry opened %d pull requests with reports %q", len(gh.prs), reportsPosted(gh)) + } + }) + t.Run("a new batch whose report comment failed", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 || !strings.Contains(stderr, "the body keeps it") { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + first := bodyReportOf(t, gh.find(90).Body) + mustRun(t, f, gh, change(f, gh)...) + second := bodyReportOf(t, gh.find(90).Body) + want, err := reportComment(first) + if err != nil || len(gh.issueComments) != 2 || gh.issueComments[0].Body != want || reportsPosted(gh)[1] != second.doc.ID || first.doc.ID == second.doc.ID { + t.Errorf("comments %q; want the first report replayed as it was, then the retry's", reportsPosted(gh)) + } + }) + t.Run("a continuation whose push failed", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + first := reportsPosted(gh)[0] + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + f.refusePushes(true) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + f.refusePushes(false) + attempted := reportsPosted(gh)[1] + if bodyReportOf(t, gh.find(90).Body).doc.ID != first { + t.Error("the body moved on although the push failed") + } + mustRun(t, f, gh, change(f, gh)...) + once(t, gh, first, attempted) + if len(reportsPosted(gh)) != 3 { + t.Errorf("reports %q; want the first, the attempted and the retry's", reportsPosted(gh)) + } + }) + t.Run("a continuation whose body edit failed", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + first := reportsPosted(gh)[0] + before := f.remoteHead(batchBranch1) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.fail = map[string]error{"editPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + attempted := reportsPosted(gh)[1] + if f.remoteHead(batchBranch1) == before || bodyReportOf(t, gh.find(90).Body).doc.ID != first { + t.Error("the push or the body is not where the failure left them") + } + mustRun(t, f, gh, change(f, gh)...) + once(t, gh, first, attempted) + if last := reportsPosted(gh)[len(reportsPosted(gh))-1]; bodyReportOf(t, gh.find(90).Body).doc.ID != last || len(reportsPosted(gh)) != 3 { + t.Errorf("reports %q; want the body to carry the retry's, posted last", reportsPosted(gh)) + } + }) + t.Run("a batch closed while the run writes to it", func(t *testing.T) { + f, gh := setup(t) + openBatch(t, f, gh) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.after = map[string]func(){"createComment": func() { gh.find(90).State = "closed" }} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || len(gh.edited) != 0 { + t.Errorf("run = %d, edited %q; want 2 and no edit of the closed pull request:\n%s", code, gh.edited, stderr) + } + }) + t.Run("a batch closed before its report was posted", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + kept := bodyReportOf(t, gh.find(90).Body) + gh.find(90).State = "closed" + gh.nextPR = 91 + mustRun(t, f, gh, change(f, gh)...) + if slices.Contains(gh.commented, 90) || slices.Contains(reportsPosted(gh), kept.doc.ID) { + t.Errorf("commented on %v, posting %q; want nothing on the closed #90", gh.commented, reportsPosted(gh)) + } + code, stdout, stderr := f.exec(gh, "report", "--pr", "90") + if code != 0 || !strings.Contains(stdout, "report "+kept.doc.ID) || !strings.Contains(stdout, "kept in the body only") { + t.Errorf("report --pr 90 = %d:\n%s%s", code, stdout, stderr) + } + }) + // Every comment is posted only after GitHub says the batch is still + // open, the replay's included, so a close between the two stops the + // second. + t.Run("a batch closed between the replay and the new report", func(t *testing.T) { + f, gh := setup(t) + gh.fail = map[string]error{"createComment": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + kept := bodyReportOf(t, gh.find(90).Body) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.after = map[string]func(){"createComment": func() { gh.find(90).State = "closed" }} + code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push") + if code != 2 || !strings.Contains(stderr, "batch pull request #90 is closed now; nothing is written to it") || !slices.Equal(reportsPosted(gh), []string{kept.doc.ID}) || len(gh.edited) != 0 { + t.Errorf("run = %d, posted %q, edited %q; want 2 after the replay alone:\n%s", code, reportsPosted(gh), gh.edited, stderr) + } + }) +} + +// The reader says how a report stands against the pull request as GitHub +// shows it now, and claims publication only when the head or the body shows +// it: a failed push, a push whose answer was lost, a body that was not +// updated and a rebuild since all read as they are. +func TestReportPublicationStatus(t *testing.T) { + setup := func(t *testing.T) (*fixture, *fakeGH, string) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + return f, gh, reportsPosted(gh)[0] + } + status := func(t *testing.T, f *fixture, gh *fakeGH, id, want string) { + t.Helper() + code, stdout, stderr := f.exec(gh, "report", "--pr", "90", "--id", id) + if code != 0 || !strings.Contains(stdout, want) { + t.Errorf("report --id %.12s = %d, want %q:\n%s%s", id, code, want, stdout, stderr) + } + } + t.Run("a push the remote refused, and one it took whose answer was lost", func(t *testing.T) { + f, gh, first := setup(t) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + f.refusePushes(true) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.refusePushes(false) + attempted := reportsPosted(gh)[1] + status(t, f, gh, attempted, "publication unconfirmed: #90's head is ") + status(t, f, gh, attempted, "its body carries report "+first[:12]) + // The same head reaching the remote is what a push GitHub took looks + // like when its answer never came back. + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + status(t, f, gh, attempted, "the batch this report describes, but its body carries report "+first[:12]) + }) + t.Run("a push followed by a failed body edit", func(t *testing.T) { + f, gh, first := setup(t) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh.fail = map[string]error{"editPull": errString("HTTP 502")} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + status(t, f, gh, reportsPosted(gh)[1], "the batch this report describes, but its body carries report "+first[:12]) + status(t, f, gh, first, "#90's body carries this report") + }) + t.Run("a rebuild since", func(t *testing.T) { + f, gh, first := setup(t) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "x", "--push") + second := reportsPosted(gh)[1] + status(t, f, gh, first, "publication unconfirmed: #90's head is ") + status(t, f, gh, second, "#90's body carries this report") + }) +} + +// forged is a report on release-1.1 with what change sets, under an ID that +// matches it: only checking what it says stands in its way. +func forged(change func(d *reportDoc)) storedReport { + sha := strings.Repeat("a", 40) + d := reportDoc{Format: reportFormat, Line: "release-1.1", Diag: map[string][]string{}, Other: []string{sha}, + Built: builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeNew}}} + d.Primary.Picked = []int{5} + change(&d) + d.ID = reportID(d) + return storedReport{doc: d, raw: reportJSON(d)} +} + +// reportRun runs report with gh faked; it reads nothing but GitHub. +func reportRun(gh *fakeGH, args ...string) (int, string, string) { + var stdout, stderr strings.Builder + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 0, 0, 0, time.UTC) }} + code := a.main(append([]string{"report"}, args...)) + return code, stdout.String(), stderr.String() +} + +// A report is read only when everything it says is what a run writes, and +// it is for the line of the pull request it sits on: an ID that matches its +// content proves only that nobody changed it since it was hashed. +func TestReportRefusesWhatItCannotVouchFor(t *testing.T) { + for name, change := range map[string]func(d *reportDoc){ + "a line that is no release line": func(d *reportDoc) { d.Line = "release-1.1\n\x1b[2Jpicked" }, + "another line": func(d *reportDoc) { d.Line = "release-1.0" }, + "a main that is no commit": func(d *reportDoc) { d.Built.Main = "main\n#5 picked" }, + "a time that is none": func(d *reportDoc) { d.Built.At = "2026-10-26T" }, + "an unknown operation": func(d *reportDoc) { d.Built.Options.Mode = "new\x1b[2J" }, + "a run that is no Actions run": func(d *reportDoc) { d.Built.Run = "javascript:alert(1)" }, + "a tool that is no commit": func(d *reportDoc) { d.Built.Tool = "v1\n" }, + "a commit on main that is none": func(d *reportDoc) { d.Other = []string{"HEAD\x1b]0;x\x07"} }, + "a deferred commit that is none": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 6, Commit: "-x", Files: []string{"a"}}} + }, + "an original named twice": func(d *reportDoc) { d.Primary.Missing = []int{5} }, + "an original that is none": func(d *reportDoc) { d.Primary.Skipped.Type = []int{-1} }, + "a diagnostic of two lines": func(d *reportDoc) { d.Diag = map[string][]string{"5": {"incomplete\n\x1b[2Jforged"}} }, + "a diagnostic of no original": func(d *reportDoc) { d.Diag = map[string][]string{"five": {"incomplete"}} }, + // Fields that speak of one another have to agree. + "a drop category with no drop": func(d *reportDoc) { d.Primary.Skipped.Dropped = []int{6} }, + "a stale original it has missing": func(d *reportDoc) { + d.Primary.Missing, d.Overlays.Stale = []int{7}, []int{7} + }, + "a stale original it puts nowhere": func(d *reportDoc) { d.Overlays.Stale = []int{8} }, + "a changed original it defers": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 9, Files: []string{}, Reason: "fails the e2e"}} + d.Overlays.Changed = []int{9} + }, + "a skip label on a picked original": func(d *reportDoc) { d.Overrides.Skip = []int{5} }, + "a skip-label category with no label": func(d *reportDoc) { d.Primary.Skipped.SkipLabel = []int{10} }, + "an include label on a rule 2 skip": func(d *reportDoc) { + d.Primary.Skipped.Type, d.Overrides.Include = []int{11}, []int{11} + }, + "both labels on one original": func(d *reportDoc) { + d.Primary.Skipped.SkipLabel, d.Overrides.Include, d.Overrides.Skip = []int{12}, []int{12}, []int{12} + }, + "a label on no original": func(d *reportDoc) { d.Overrides.Include = []int{13} }, + "a diagnostic of an original it puts nowhere": func(d *reportDoc) { d.Diag = map[string][]string{"14": {"incomplete"}} }, + "a workflow wait without the flag": func(d *reportDoc) { + d.Primary.Deferred = []deferredEntry{{Original: 15, Files: []string{".github/workflows/ci.yaml"}, Workflows: true}} + }, + "a close that still picks": func(d *reportDoc) { d.Built.Options.Mode = modeClose }, + "a parked original": func(d *reportDoc) { d.Primary.Parked = []json.RawMessage{json.RawMessage(`{"original":16}`)} }, + } { + s := forged(change) + c, _ := reportComment(storedReport{doc: forged(func(*reportDoc) {}).doc, raw: s.raw}) + comment := issueComment{ID: 1, Body: strings.Replace(c, forged(func(*reportDoc) {}).doc.ID, s.doc.ID, 1)} + comment.User.Login, comment.User.Type = "maintainer", "User" + for where, gh := range map[string]*fakeGH{ + "in the body": {prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(s.raw)+"\n")}}, + "in a comment": {prs: []pullRequest{notifyPR("20261026", "text")}, issueComments: []issueComment{comment}}, + } { + if code, stdout, stderr := reportRun(gh, "--pr", "90", "--id", s.doc.ID); code != 2 || stdout != "" { + t.Errorf("%s, %s: report = %d, want 2 and nothing printed:\n%s%s", name, where, code, stdout, stderr) + } + } + } + if code, stdout, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(forged(func(*reportDoc) {}).raw)+"\n")}}, "--pr", "90"); code != 0 || !strings.Contains(stdout, "#5 picked") { + t.Errorf("report of a sound one = %d:\n%s%s", code, stdout, stderr) + } +} + +// What report prints stays one line per field and carries nothing a +// terminal acts on, whatever reached it. +func TestReportPrintsNothingThatDrivesATerminal(t *testing.T) { + const evil = "\n\x1b[2Jfake: #9 picked\u202e" + s := forged(func(d *reportDoc) { + d.Line, d.Built.At, d.Built.Options.Mode = "release-1.1"+evil, "2026-10-26"+evil, "new"+evil + d.Built.Main, d.Built.Tool, d.Built.Run, d.Built.Newer = "main"+evil, "tool"+evil, "run"+evil, "newer"+evil + d.Other = []string{"other" + evil} + d.Primary.Deferred = []deferredEntry{{Original: 6, Reason: "why" + evil}} + }) + var stdout strings.Builder + a := &app{stdout: &stdout} + pr := notifyPR("20261026", "") + a.printReport(&pr, s, nil, true) + for _, ln := range strings.Split(stdout.String(), "\n") { + if strings.HasPrefix(ln, "fake") || strings.ContainsFunc(ln, func(r rune) bool { return r != '\t' && unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) { + t.Errorf("report printed %q", ln) + } + } +} + +// The comment is made of what was checked: a line, a time and an ID that +// carry no markup into the summary GitHub renders. +func TestReportCommentIsBuiltFromCheckedFields(t *testing.T) { + for name, change := range map[string]func(d *reportDoc){ + "markup in the line": func(d *reportDoc) { d.Line = "release-1.1" }, + "markup in the time": func(d *reportDoc) { d.Built.At = "2026-10-26T06:00:00Z" }, + "a mention": func(d *reportDoc) { d.Line = "release-1.1 @cozystack/maintainers" }, + } { + if c, err := reportComment(forged(change)); err == nil { + t.Errorf("%s: reportComment made\n%s", name, c) + } + } + if _, err := reportComment(forged(func(*reportDoc) {})); err != nil { + t.Errorf("reportComment of a sound report: %v", err) + } +} + +// Anyone can comment on a public pull request: the reader takes a report +// from a comment only when the batch trusts its author, whatever ID it +// carries. +func TestReportReaderTakesOnlyTrustedComments(t *testing.T) { + s := forged(func(*reportDoc) {}) + c, err := reportComment(s) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + login, kind string + read bool + }{ + {"someone", "User", false}, + {"renovate[bot]", "Bot", false}, + {"github-actions[bot]", "Bot", false}, + {"maintainer", "User", true}, + {defaultBatchBot, "Bot", true}, + } { + comment := issueComment{ID: 1, Body: c} + comment.User.Login, comment.User.Type = tc.login, tc.kind + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", "text")}, issueComments: []issueComment{comment}, bot: defaultBatchBot} + code, stdout, stderr := reportRun(gh, "--pr", "90", "--id", s.doc.ID) + if read := code == 0 && strings.Contains(stdout, "#5 picked"); read != tc.read || !tc.read && (code != 2 || !strings.Contains(stderr, "keeps no report")) { + t.Errorf("a report by %s: report = %d, read %v, want %v:\n%s%s", tc.login, code, read, tc.read, stdout, stderr) + } + } +} + +// A report whose ID is not one says so without repeating it as written. +func TestReportErrorsRepeatNothingRaw(t *testing.T) { + s := forged(func(*reportDoc) {}) + // Short enough that a cut to 12 characters keeps the forged line. + raw := strings.Replace(s.raw, `"id":"`+s.doc.ID+`"`, `"id":"\nforged"`, 1) + code, _, stderr := reportRun(&fakeGH{prs: []pullRequest{notifyPR("20261026", "text\n\n"+reportBlock(raw)+"\n")}}, "--pr", "90") + for _, ln := range strings.Split(stderr, "\n") { + if strings.HasPrefix(ln, "forged") { + t.Errorf("report = %d, printed the forged line %q", code, ln) + } + } + if code != 2 { + t.Errorf("report = %d, want 2:\n%s", code, stderr) + } +} diff --git a/cmd/backport-batch/rules.go b/cmd/backport-batch/rules.go new file mode 100644 index 000000000..e7aa2740a --- /dev/null +++ b/cmd/backport-batch/rules.go @@ -0,0 +1,344 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "maps" + "path/filepath" + "regexp" + "slices" + "strings" + + "sigs.k8s.io/yaml" +) + +// Rule numbers as docs/release/backports.md numbers them. Rule 5, a clean +// pick, is decided while the batch is assembled. +const ( + ruleOnLine = 1 + ruleType = 2 + ruleBreaking = 3 + ruleTouches = 4 + ruleNewer = 6 + // Not a rule of the policy: the backport/skip override. + ruleSkipLabel = 100 + // Rule 1 for a line that started under the policy after it branched: + // merged before that start. + ruleBeforeStart = 101 + // Not a rule of the policy: a maintainer dropped the original from the + // line with run --drop. + ruleDropped = 102 +) + +const ( + // The numbered scripts run-migrations.sh walks; hack/check-migrations-target.sh + // reads the same directory. + migrationsDir = "packages/core/platform/images/migrations/migrations/" + // Where migrations.targetVersion is set; hack/check-migrations-target.sh + // reads it from here. + platformValues = "packages/core/platform/values.yaml" + // Only a file holding this text at either end of the change can hold a + // CRD document; crdDocs decides document by document. + crdKind = "CustomResourceDefinition" +) + +// The title grammar and the breaking footer are pr-labeler.yaml's, so that a +// pull request labelled kind/bug and kind/breaking-change reads the same here. +var ( + backportPrefixRE = regexp.MustCompile(`^(?:\[Backport [^\]]+\]\s+)+`) + conventionalRE = regexp.MustCompile(`^([a-z]+)(?:\(([^)]+)\))?(!)?:\s*\S`) + breakingFooterRE = regexp.MustCompile(`(?m)^BREAKING[ -]CHANGE:`) + htmlCommentRE = regexp.MustCompile(`(?s)`) + targetVersionRE = regexp.MustCompile(`(?m)^[+-][ \t]*targetVersion[ \t]*:`) +) + +// title is a parsed Conventional Commits title. +type title struct { + typ string + scopes []string + bang bool +} + +func parseTitle(s string) (title, bool) { + m := conventionalRE.FindStringSubmatch(backportPrefixRE.ReplaceAllString(s, "")) + if m == nil { + return title{}, false + } + t := title{typ: m[1], bang: m[3] == "!"} + for scope := range strings.SplitSeq(m[2], ",") { + if scope = strings.TrimSpace(scope); scope != "" { + t.scopes = append(t.scopes, scope) + } + } + return t, true +} + +// typeRule is rule 2 together with the policy's "never in a patch release": +// dependency bumps, scope deps, stay out whatever their type. +func typeRule(s string) string { + t, ok := parseTitle(s) + switch { + case !ok: + return "the title is not a Conventional Commits title" + case !slices.Contains(takenTypes, t.typ): + return "type " + t.typ + case slices.ContainsFunc(t.scopes, func(s string) bool { return slices.Contains(excludedScopes, s) }): + return "an excluded scope" + } + return "" +} + +// breakingRule is rule 3. The PR template quotes the footer inside an HTML +// comment, which a body merged with the template left in carries. +func breakingRule(s, body string) string { + if t, ok := parseTitle(s); ok && t.bang { + return "! in the title" + } + if breakingFooterRE.MatchString(htmlCommentRE.ReplaceAllString(body, "")) { + return "BREAKING CHANGE footer" + } + return "" +} + +// fileChange is one path a merge changes against its first parent. +type fileChange struct { + status byte + path string +} + +func changedFiles(from, to string) ([]fileChange, error) { + out, err := git("diff", "--no-ext-diff", "--no-textconv", "--name-status", "-z", "--no-renames", "--end-of-options", from, to) + if err != nil { + return nil, err + } + f := splitNUL(out) + if len(f)%2 != 0 { + return nil, fmt.Errorf("unreadable diff of %s..%s", from, to) + } + var changes []fileChange + for i := 0; i < len(f); i += 2 { + changes = append(changes, fileChange{status: f[i][0], path: f[i+1]}) + } + return changes, nil +} + +func paths(changes []fileChange) []string { + out := make([]string, 0, len(changes)) + for _, c := range changes { + out = append(out, c.path) + } + return out +} + +const ( + touchesMigrations = "platform migrations" + touchesTargetVersion = "targetVersion" + touchesAPI = "a sizeable API change" + touchesCRD = "a CRD" +) + +// touchesRule is rule 4: what of migrations, targetVersion, the API and CRDs +// the merge touches, or "" when it touches none of them. +func touchesRule(o *original, changes []fileChange) (string, error) { + hit := map[string]bool{} + for _, c := range changes { + p := c.path + switch { + case strings.HasPrefix(p, migrationsDir): + hit[touchesMigrations] = true + case p == platformValues: + diff, err := git("--literal-pathspecs", "diff", "-U0", "--no-ext-diff", "--no-textconv", "--end-of-options", o.merge+"^1", o.merge, "--", platformValues) + if err != nil { + return "", err + } + hit[touchesTargetVersion] = targetVersionRE.MatchString(diff) + } + } + var err error + if hit[touchesCRD], err = changesCRD(o.merge+"^1", o.merge, changes); err != nil { + return "", err + } + asks, err := readsAPI(o.merge+"^1", o.merge, changes) + if err != nil { + return "", err + } + apiDetail := "" + if asks { + if apiDetail, err = sizeableAPIChange(o); err != nil { + return "", err + } + hit[touchesAPI] = apiDetail != "" + } + var why []string + for _, r := range []string{touchesMigrations, touchesTargetVersion, touchesAPI, touchesCRD} { + switch { + case !hit[r]: + case r == touchesAPI: + why = append(why, r+" ("+apiDetail+")") + default: + why = append(why, r) + } + } + return strings.Join(why, ", "), nil +} + +// changesCRD reports whether the change adds, removes or modifies a +// CustomResourceDefinition document. It looks document by document: an edit to +// the DaemonSet beside a CRD in one manifest is not a CRD change, and a CRD in +// a test fixture changes nothing a cluster runs. +func changesCRD(parent, merge string, changes []fileChange) (bool, error) { + var files []string + for _, c := range changes { + if manifest(c.path) && !testFixture(c.path) { + files = append(files, c.path) + } + } + candidates := map[string]bool{} + for _, rev := range []string{parent, merge} { + found, err := grepFiles(rev, files, "-I", "-e", crdKind) + if err != nil { + return false, err + } + for _, f := range found { + candidates[f] = true + } + } + for _, c := range changes { + if !candidates[c.path] { + continue + } + var before, after map[string]string + if c.status != 'A' { + content, err := git("cat-file", "blob", parent+":"+c.path) + if err != nil { + return false, err + } + before = crdDocs(content) + } + if c.status != 'D' { + content, err := git("cat-file", "blob", merge+":"+c.path) + if err != nil { + return false, err + } + after = crdDocs(content) + } + if !maps.Equal(before, after) { + return true, nil + } + } + return false, nil +} + +// manifest reports whether path is a file a chart or a cluster can apply: YAML, +// JSON or a Helm template. A CRD quoted in documentation or code applies +// nothing. +func manifest(path string) bool { + switch filepath.Ext(path) { + case ".yaml", ".yml", ".json", ".tpl": + return true + } + dirs := strings.Split(path, "/") + return slices.ContainsFunc(dirs[:len(dirs)-1], func(d string) bool { return d == "templates" || d == "crds" }) +} + +func testFixture(path string) bool { + dirs := strings.Split(path, "/") + return slices.ContainsFunc(dirs[:len(dirs)-1], func(d string) bool { return d == "tests" || d == "testdata" }) +} + +// grepFiles lists which of files at rev hold any of the fixed strings that +// patterns give. Paths are passed literally, so a name such as ":(exclude)x" +// means only that file. +func grepFiles(rev string, files []string, patterns ...string) ([]string, error) { + const chunk = 500 + var found []string + for len(files) > 0 { + n := min(chunk, len(files)) + args := append([]string{"--literal-pathspecs", "grep", "-l", "-z", "-F"}, patterns...) + args = append(append(args, "--end-of-options", rev, "--"), files[:n]...) + stdout, stderr, code, err := gitStatus(args...) + switch { + case err != nil: + return nil, err + case code != 0 && code != 1: + return nil, fmt.Errorf("git grep at %s: exit %d: %s", rev, code, strings.TrimSpace(stderr)) + } + for _, hit := range splitNUL(stdout) { + if path, ok := strings.CutPrefix(hit, rev+":"); ok { + found = append(found, path) + } + } + files = files[n:] + } + return found, nil +} + +var ( + docSeparatorRE = regexp.MustCompile(`(?m)^---[ \t]*(?:#.*)?$`) + // A document that declares the kind on a line of its own, in YAML or in + // JSON, under a bare or quoted key, is a CRD even when it cannot be read; + // a sentence or a lookup call that names it declares nothing. + crdKindLineRE = regexp.MustCompile(`(?m)^[ \t]*(?:-[ \t]+)?(?:kind|"kind"|'kind')[ \t]*:[ \t]*["']?CustomResourceDefinition["']?[ \t]*,?[ \t]*(?:#.*)?$`) +) + +// crdDocs maps the CRD documents of a file to their text, keyed by +// metadata.name, so a CRD added, removed or edited changes the map. Every +// document that names the kind is read, so a CRD in compact JSON or under a +// quoted key counts as well. A document that declares the kind on a line of +// its own but cannot be read as an object, a CRD behind a Helm template guard +// for one, is kept under its own text, so that any change to it counts: the +// check fails closed. So does a list holding a CRD. +func crdDocs(content string) map[string]string { + docs := map[string]string{} + for _, doc := range docSeparatorRE.Split(content, -1) { + declares := crdKindLineRE.MatchString(doc) + if !declares && !strings.Contains(doc, crdKind) { + continue + } + var head struct { + Kind string `json:"kind"` + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Items []struct { + Kind string `json:"kind"` + } `json:"items"` + } + key := "" + switch err := yaml.Unmarshal([]byte(doc), &head); { + case err != nil && declares: + key = "unreadable:" + doc + case err != nil: + continue + case head.Kind == crdKind && head.Metadata.Name != "": + key = "crd:" + head.Metadata.Name + case head.Kind == crdKind: + key = "unnamed:" + doc + case strings.HasSuffix(head.Kind, "List") && slices.ContainsFunc(head.Items, func(i struct { + Kind string `json:"kind"` + }) bool { + return i.Kind == crdKind + }): + key = "list:" + doc + default: + continue + } + docs[key] += doc + } + return docs +} diff --git a/cmd/backport-batch/rules_test.go b/cmd/backport-batch/rules_test.go new file mode 100644 index 000000000..ca1cd58a2 --- /dev/null +++ b/cmd/backport-batch/rules_test.go @@ -0,0 +1,461 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "maps" + "slices" + "strings" + "testing" +) + +func TestTypeRule(t *testing.T) { + cases := []struct { + title string + taken bool + }{ + {"fix(postgres): keep the replica count on resize", true}, + {"fix: typo in monitoring-agents values.yaml", true}, + {"security(dashboard): escape the tenant name", true}, + {"fix(ci): retry the flaky e2e step", true}, + {"[Backport release-1.6] fix(linstor): x", true}, + {"[Backport release-1.5] [Backport release-1.6] fix(linstor): x", true}, + {"fix(platform, dashboard): two scopes", true}, + {"feat(postgres): add a field", false}, + {"chore(release): promote", false}, + {"docs(release): split the page", false}, + {"refactor(api): move code", false}, + {"fix(deps): bump golang.org/x/net", false}, + {"fix(platform, deps): bump and fix", false}, + {"Fix(postgres): capitalised type", false}, + {"fix postgres without a colon", false}, + {"[postgres] bracket form", false}, + {"fixes(postgres): not the fix type", false}, + {"", false}, + } + for _, tc := range cases { + reason := typeRule(tc.title) + if (reason == "") != tc.taken { + t.Errorf("typeRule(%q) = %q, want taken=%v", tc.title, reason, tc.taken) + } + } +} + +func TestBreakingRule(t *testing.T) { + cases := []struct { + name, title, body string + breaking bool + }{ + {"plain fix", "fix(x): y", "## What this PR does\n\nFixes it.", false}, + {"bang", "fix(x)!: y", "", true}, + {"bang without scope", "fix!: y", "", true}, + {"bang behind a backport prefix", "[Backport release-1.6] fix(x)!: y", "", true}, + {"footer", "fix(x): y", "Body.\n\nBREAKING CHANGE: the field is gone", true}, + {"hyphenated footer", "fix(x): y", "BREAKING-CHANGE: the field is gone", true}, + {"footer quoted mid-line", "fix(x): y", "This is not a BREAKING CHANGE: nothing changes.", false}, + {"footer inside the template comment", "fix(x): y", "\nBody.", false}, + {"lower-case footer", "fix(x): y", "breaking change: no", false}, + } + for _, tc := range cases { + reason := breakingRule(tc.title, tc.body) + if (reason != "") != tc.breaking { + t.Errorf("%s: breakingRule = %q, want breaking=%v", tc.name, reason, tc.breaking) + } + } +} + +// Rule 4 reads the merge itself, so each case is a pull request merged into a +// real repository. +func TestTouchesRule(t *testing.T) { + cases := []struct { + name string + setup map[string]string // committed on main before the pull request + files map[string]string + want string + }{ + {name: "a chart change", files: map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}}, + {name: "a migration script", files: map[string]string{migrationsDir + "61": "#!/bin/sh\n"}, want: "platform migrations"}, + {name: "a migration library file", files: map[string]string{migrationsDir + "lib/common.sh": "x\n"}, want: "platform migrations"}, + {name: "the migration runner is not a migration", files: map[string]string{"packages/core/platform/images/migrations/run-migrations.sh": "x\n"}}, + { + name: "targetVersion moves", + setup: map[string]string{platformValues: "migrations:\n enabled: false\n targetVersion: 60\n"}, + files: map[string]string{platformValues: "migrations:\n enabled: false\n targetVersion: 61\n"}, + want: "targetVersion", + }, + { + name: "another key of the platform values", + setup: map[string]string{platformValues: "migrations:\n enabled: false\n targetVersion: 60\n"}, + files: map[string]string{platformValues: "migrations:\n enabled: true\n targetVersion: 60\n"}, + }, + {name: "api types the gate sees nothing in", setup: demoAPI, files: map[string]string{"api/apps/v1alpha1/types.go": "package v1alpha1\n"}}, + {name: "pkg/apis is not api/", files: map[string]string{"pkg/apis/apps/types.go": "package apps\n"}}, + {name: "a CRD added under crds/", files: map[string]string{"packages/system/x/charts/x/crds/a.yaml": crdDoc("as.x.io", "a") + "\n"}, want: "a CRD"}, + {name: "a ConfigMap under crds/ is not a CRD", files: map[string]string{"packages/system/x/charts/x/crds/a.yaml": "kind: ConfigMap\n"}}, + { + name: "a DaemonSet edit next to an unchanged CRD", + setup: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("as.x.io", "a"), daemonSet("1"))}, + files: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("as.x.io", "a"), daemonSet("2"))}, + }, + { + name: "a real CRD edit in a manifest with other documents", + setup: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("as.x.io", "a"), daemonSet("1"))}, + files: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("as.x.io", "b"), daemonSet("1"))}, + want: "a CRD", + }, + { + name: "a CRD renamed", + setup: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("as.x.io", "a"), daemonSet("1"))}, + files: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(crdDoc("bs.x.io", "a"), daemonSet("1"))}, + want: "a CRD", + }, + {name: "a CRD in a fixture under tests/", files: map[string]string{"packages/system/x/tests/render_test.yaml": crdDoc("as.x.io", "a")}}, + {name: "a CRD in a fixture under testdata/", files: map[string]string{"hack/testdata/crd.yaml": crdDoc("as.x.io", "a")}}, + { + name: "a templated CRD edited, which cannot be read", + setup: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(templatedCRD("a"), daemonSet("1"))}, + files: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(templatedCRD("b"), daemonSet("1"))}, + want: "a CRD", + }, + { + name: "a DaemonSet edit next to an unchanged templated CRD", + setup: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(templatedCRD("a"), daemonSet("1"))}, + files: map[string]string{"packages/system/x/templates/m.yaml": multiDoc(templatedCRD("a"), daemonSet("2"))}, + }, + { + name: "an unparseable file that names the kind, changed", + setup: map[string]string{"x/broken.yaml": "kind: CustomResourceDefinition\nspec: [unclosed\n"}, + files: map[string]string{"x/broken.yaml": "kind: CustomResourceDefinition\nspec: [still unclosed\n"}, + want: "a CRD", + }, + { + name: "a lookup that names the kind in a Helm helper", + setup: map[string]string{"packages/system/x/templates/_helpers.tpl": "{{- $crd := lookup \"apiextensions.k8s.io/v1\" \"CustomResourceDefinition\" \"\" .crd -}}\n{{- if $crd }}a{{ end }}\n"}, + files: map[string]string{"packages/system/x/templates/_helpers.tpl": "{{- $crd := lookup \"apiextensions.k8s.io/v1\" \"CustomResourceDefinition\" \"\" .crd -}}\n{{- if $crd }}b{{ end }}\n"}, + }, + { + name: "a CRD example in documentation", + setup: map[string]string{"docs/crds.md": "Example:\n\n```yaml\nkind: CustomResourceDefinition\nmetadata:\n name: as.x.io\n```\n"}, + files: map[string]string{"docs/crds.md": "An example:\n\n```yaml\nkind: CustomResourceDefinition\nmetadata:\n name: bs.x.io\n```\n"}, + }, + {name: "a CRD in a template without a YAML extension", files: map[string]string{"packages/system/x/templates/crd.tmpl": crdDoc("as.x.io", "a")}, want: "a CRD"}, + {name: "a CRD added as JSON", files: map[string]string{"x/crd.json": `{"apiVersion": "apiextensions.k8s.io/v1",` + "\n" + ` "kind": "CustomResourceDefinition",` + "\n" + ` "metadata": {"name": "as.x.io"}}` + "\n"}, want: "a CRD"}, + {name: "a List of CRDs", files: map[string]string{"x/list.yaml": "kind: List\nitems:\n- kind: CustomResourceDefinition\n metadata:\n name: as.x.io\n"}, want: "a CRD"}, + {name: "a template declaring a CRD", files: map[string]string{"packages/system/x/templates/crd.yaml": "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\n"}, want: "a CRD"}, + {name: "a quoted declaration", files: map[string]string{"x/crd.yaml": " kind: \"CustomResourceDefinition\" # yes\n"}, want: "a CRD"}, + { + name: "a CRD changed in place", + setup: map[string]string{"x/crd.yaml": "kind: CustomResourceDefinition\nspec: {}\n"}, + files: map[string]string{"x/crd.yaml": "kind: CustomResourceDefinition\nspec: {a: 1}\n"}, + want: "a CRD", + }, + { + name: "a CRD deleted", + setup: map[string]string{"x/crd.yaml": "kind: CustomResourceDefinition\n"}, + files: map[string]string{"x/crd.yaml": ""}, + want: "a CRD", + }, + { + name: "a CRD turned into something else", + setup: map[string]string{"x/crd.yaml": "kind: CustomResourceDefinition\n"}, + files: map[string]string{"x/crd.yaml": "kind: ConfigMap\n"}, + want: "a CRD", + }, + {name: "a CRD list is not a CRD", files: map[string]string{"x/list.yaml": "kind: CustomResourceDefinitionList\n"}}, + {name: "a CRD as compact JSON", files: map[string]string{"x/crd.json": `{"apiVersion":"apiextensions.k8s.io/v1","kind":"CustomResourceDefinition","metadata":{"name":"as.x.io"}}` + "\n"}, want: "a CRD"}, + {name: "a CRD under a quoted kind key", files: map[string]string{"x/crd.yaml": "'kind': CustomResourceDefinition\nmetadata:\n name: as.x.io\n"}, want: "a CRD"}, + { + name: "a templated CRD under a quoted kind key, edited", + setup: map[string]string{"packages/system/x/templates/m.yaml": strings.Replace(templatedCRD("a"), "kind:", "'kind':", 1)}, + files: map[string]string{"packages/system/x/templates/m.yaml": strings.Replace(templatedCRD("b"), "kind:", "'kind':", 1)}, + want: "a CRD", + }, + {name: "a ConfigMap as compact JSON naming the kind", files: map[string]string{"x/cm.json": `{"kind":"ConfigMap","data":{"note":"kind: CustomResourceDefinition"}}` + "\n"}}, + {name: "a CRD mentioned in prose", files: map[string]string{"x/README.md": "Set kind: CustomResourceDefinition here.\n"}}, + {name: "a file named like an option declaring a CRD", files: map[string]string{"-e.yaml": "kind: CustomResourceDefinition\n"}, want: "a CRD"}, + // As pathspec magic, :(icase)... would match a file of that name in + // another case, and there is none. + {name: "a file named like pathspec magic declaring a CRD", files: map[string]string{":(icase)zz-crd.yaml": "kind: CustomResourceDefinition\n"}, want: "a CRD"}, + { + name: "a migration and a sizeable API change together", + setup: demoAPI, + files: map[string]string{migrationsDir + "61": "x\n", "api/x.go": "package x\n", demoRD: cozyrd("demo", `{"type":"object","properties":{}}`)}, + want: "platform migrations, a sizeable API change (breaking-change: [v1alpha1] spec: field \"size\" was removed)", + }, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + setup := maps.Clone(apiBase) + maps.Copy(setup, tc.setup) + f.squash(1000, "chore: setup", "", setup) + mc := f.pr(10+i, "fix(x): y", "", tc.files) + o := &original{number: 10 + i, merge: mc, parents: 2} + changes, err := changedFiles(mc+"^1", mc) + if err != nil { + t.Fatal(err) + } + got, err := touchesRule(o, changes) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("touchesRule = %q, want %q", got, tc.want) + } + }) + } +} + +// Rule 6: an older line takes only what the newer line already carries, and +// rule 1: neither line takes what it carries already. +func TestNewerLineAndOnLine(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(2, "fix(app): before nothing", "", map[string]string{"app/a.txt": "a\n"}) + f.pr(3, "fix(app): picked onto 1.1", "", map[string]string{"app/b.txt": "b\n"}) + mc3 := f.git("rev-parse", "HEAD") + f.pr(4, "fix(app): only on main", "", map[string]string{"app/c.txt": "c\n"}) + f.onLine("release-1.1", "fix(app): picked onto 1.1 (#3)\n\n(cherry picked from commit "+mc3+")", map[string]string{"app/b.txt": "b\n"}) + + for _, tc := range []struct { + line string + qualified []int + skipped map[int]int + }{ + {line: "release-1.1", qualified: []int{2, 4}, skipped: map[int]int{3: ruleOnLine}}, + // #1 is a feature; #2 and #4 are not on release-1.1 yet; #3 is. + {line: "release-1.0", qualified: []int{3}, skipped: map[int]int{1: ruleType, 2: ruleNewer, 4: ruleNewer}}, + } { + t.Run(tc.line, func(t *testing.T) { + f.git("fetch", "-q", "origin") + l, _ := parseLine(tc.line) + b, err := (&app{gh: &fakeGH{}, stderr: &strings.Builder{}}).prepare(l) + if err != nil { + t.Fatal(err) + } + var got []int + for _, v := range b.qualifying() { + got = append(got, v.o.number) + } + if !slices.Equal(got, tc.qualified) { + t.Errorf("qualifying = %v, want %v", got, tc.qualified) + } + for n, rule := range tc.skipped { + if v := b.find(n); v == nil || v.rule != rule { + t.Errorf("#%d: verdict %+v, want rule %d", n, v, rule) + } + } + }) + } +} + +// An original merged before the newer line branched is on it, even when the +// newer line has since changed the same lines and a pick would conflict. +func TestNewerLineCarriesWhatItBranchedWith(t *testing.T) { + f := newFixture(t) + f.release("release-1.0") + f.pr(2, "fix(app): two", "", map[string]string{"app/a.txt": "two\n"}) + f.release("release-1.1") + mc3 := f.pr(3, "fix(app): three", "", map[string]string{"app/a.txt": "three\n"}) + f.onLine("release-1.1", "fix(app): three (#3)\n\n(cherry picked from commit "+mc3+")", map[string]string{"app/a.txt": "three\n"}) + if v := verdictOn(t, f, nil, "release-1.0", 2); v.rule != 0 { + t.Errorf("#2 on release-1.0 = rule %d (%s), want it to qualify", v.rule, v.reason) + } +} + +// A line branched and not yet released is one the batch runs for, and rule 6 +// holds each older line to the next newer one. +func TestNewerLineInTheRCWindow(t *testing.T) { + f := newFixture(t) + f.lines() + f.release("release-1.2") + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + for _, tc := range []struct{ line, reason string }{ + {"release-1.2", ""}, + {"release-1.1", "not on release-1.2 yet"}, + {"release-1.0", "not on release-1.1 yet"}, + } { + v := verdictOn(t, f, nil, tc.line, 5) + if v.reason != tc.reason || (v.rule == 0) != (tc.reason == "") { + t.Errorf("#5 on %s = rule %d (%s), want %q", tc.line, v.rule, v.reason, tc.reason) + } + } +} + +func TestOnlySupportedLines(t *testing.T) { + f := newFixture(t) + f.release("release-0.9") + f.lines() + code, _, stderr := f.exec(nil, "check", "--line", "release-0.9") + if code != 2 || !strings.Contains(stderr, "not a line the batch runs for") { + t.Errorf("check on release-0.9 = %d (%s), want 2 and a refusal", code, stderr) + } +} + +// demoRD is an ApplicationDefinition where apigate looks for one, and demoAPI +// the tree the API cases start from. +const demoRD = "packages/system/demo-rd/cozyrds/demo.yaml" + +// apiBase is the least API a tree holds: apigate refuses a tree with none. +var apiBase = map[string]string{"packages/system/keep-rd/cozyrds/keep.yaml": cozyrd("keep", `{"type":"object"}`)} + +var demoAPI = map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{"size":{"type":"string","description":"Size."}}}`), + "api/apps/v1alpha1/demo/types.go": "package demo\n\n// Size of it.\ntype Spec struct{ Size string }\n", +} + +func cozyrd(name, schema string) string { + return "apiVersion: cozystack.io/v1alpha1\nkind: ApplicationDefinition\nmetadata:\n name: " + name + + "\nspec:\n application:\n kind: " + strings.ToUpper(name[:1]) + name[1:] + "\n plural: " + name + "s\n singular: " + name + + "\n openAPISchema: '" + schema + "'\n" +} + +// Rule 4 holds back a change to the API only when the API review gate calls it +// sizeable: a new or removed resource or group, or a breaking schema change. +func TestAPIGateRule(t *testing.T) { + typesGo := "api/apps/v1alpha1/demo/types.go" + cases := []struct { + name string + files map[string]string + want string + }{ + {name: "a description reworded", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{"size":{"type":"string","description":"The size."}}}`), + typesGo: "package demo\n\n// The size.\ntype Spec struct{ Size string }\n", + }}, + {name: "an optional field added", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{"size":{"type":"string"},"zone":{"type":"string"}}}`), + typesGo: "package demo\n\ntype Spec struct{ Size, Zone string }\n", + }}, + {name: "a field removed", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{}}`), + typesGo: "package demo\n\ntype Spec struct{}\n", + }, want: "breaking-change"}, + {name: "a field's type changed", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{"size":{"type":"integer"}}}`), + typesGo: "package demo\n\ntype Spec struct{ Size int }\n", + }, want: "breaking-change"}, + {name: "a field made required", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","required":["size"],"properties":{"size":{"type":"string"}}}`), + typesGo: "package demo\n\n// +required\ntype Spec struct{ Size string }\n", + }, want: "breaking-change"}, + {name: "a resource added", files: map[string]string{ + "packages/system/other-rd/cozyrds/other.yaml": cozyrd("other", `{"type":"object"}`), + "api/apps/v1alpha1/other/types.go": "package other\n", + }, want: "new-resource"}, + {name: "a resource removed", files: map[string]string{ + demoRD: "", + typesGo: "", + }, want: "removed"}, + {name: "a breaking schema change that does not touch api/", files: map[string]string{ + demoRD: cozyrd("demo", `{"type":"object","properties":{}}`), + }, want: "breaking-change"}, + {name: "an aggregated group registered in a chart", files: map[string]string{ + "packages/system/extra/templates/apiservice.yaml": apiService("extra.cozystack.io"), + }, want: "new-api-group"}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + setup := maps.Clone(apiBase) + maps.Copy(setup, demoAPI) + f.squash(1000, "chore: setup", "", setup) + mc := f.pr(10+i, "fix(x): y", "", tc.files) + o := &original{number: 10 + i, merge: mc, parents: 2} + changes, err := changedFiles(mc+"^1", mc) + if err != nil { + t.Fatal(err) + } + got, err := touchesRule(o, changes) + if err != nil { + t.Fatal(err) + } + if (got != "") != (tc.want != "") || !strings.Contains(got, tc.want) { + t.Errorf("touchesRule = %q, want %q", got, tc.want) + } + }) + } +} + +func apiService(group string) string { + return "apiVersion: apiregistration.k8s.io/v1\nkind: APIService\nmetadata:\n name: v1alpha1." + group + + "\nspec:\n group: " + group + "\n version: v1alpha1\n" +} + +// The API review gate is asked about exactly the changes that reach what +// apigate parses at either end of the merge. +func TestReadsAPI(t *testing.T) { + crd := crdDoc("as.x.io", "a") + cases := []struct { + name string + setup map[string]string + files map[string]string + want bool + }{ + {name: "the apiserver's registrations", files: map[string]string{apiserverGo: "package apiserver\n"}, want: true}, + {name: "an ApplicationDefinition, which carries no marker", files: map[string]string{demoRD: "kind: ApplicationDefinition\n"}, want: true}, + {name: "a CRD in a chart", files: map[string]string{"packages/system/x/crds/a.yaml": crd}, want: true}, + {name: "an APIService under internal/", files: map[string]string{"internal/x/apiservice.yml": apiService("x.cozystack.io")}, want: true}, + { + name: "a CRD turned into something else", + setup: map[string]string{"packages/system/x/crds/a.yaml": crd}, + files: map[string]string{"packages/system/x/crds/a.yaml": "kind: ConfigMap\n"}, + want: true, + }, + {name: "a binary file with a marker", files: map[string]string{"packages/system/x/a.yaml": "\x00kind: APIService\n"}, want: true}, + {name: "chart values", files: map[string]string{"packages/system/x/values.yaml": "replicas: 2\n"}}, + {name: "Go types under api/", files: map[string]string{"api/apps/v1alpha1/demo/types.go": "package demo\n"}}, + {name: "a CRD outside the roots apigate walks", files: map[string]string{"hack/crds/a.yaml": crd}}, + {name: "a CRD in a template without a YAML extension", files: map[string]string{"packages/system/x/templates/crd.tpl": crd}}, + {name: "a file beside the ApplicationDefinitions", files: map[string]string{"packages/system/demo-rd/cozyrds/sub/demo.yaml": "kind: ApplicationDefinition\n"}}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + setup := maps.Clone(apiBase) + maps.Copy(setup, tc.setup) + f.squash(1000, "chore: setup", "", setup) + mc := f.pr(10+i, "fix(x): y", "", tc.files) + changes, err := changedFiles(mc+"^1", mc) + if err != nil { + t.Fatal(err) + } + got, err := readsAPI(mc+"^1", mc, changes) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("readsAPI = %v, want %v", got, tc.want) + } + }) + } +} + +func crdDoc(name, version string) string { + return "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: " + name + + "\nspec:\n group: x.io\n versions:\n - name: v1" + version + "\n" +} + +func daemonSet(replicas string) string { + return "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: agent\nspec:\n template:\n spec:\n priority: {{ .Values.priority | default " + replicas + " }}\n" +} + +func templatedCRD(version string) string { + return "{{- if .Values.crds }}\n" + crdDoc("ts.x.io", version) + "{{- end }}\n" +} + +func multiDoc(docs ...string) string { return "---\n" + strings.Join(docs, "---\n") } diff --git a/cmd/backport-batch/shape.go b/cmd/backport-batch/shape.go new file mode 100644 index 000000000..07a5b91f1 --- /dev/null +++ b/cmd/backport-batch/shape.go @@ -0,0 +1,439 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "slices" + "strconv" + "strings" +) + +// A batch carries each original as one unit on its first-parent line. A +// merged pull request is a merge commit whose first parent is the batch before +// it and whose second parent ends a straight line of picks of the pull +// request's own commits, oldest first, each keeping its author and sign-offs; +// the merge says which merge commit on main it carries in a Backported-from +// trailer. A squash-merged original is one pick of its commit on main. + +// pickOriginal picks one original onto HEAD in that shape. A conflict on any +// of its commits takes the branch back to where it was before the original, +// so nothing of it is left half applied. Hooks are off so that each commit +// carries its message and -x line as git writes them; signing is left to the +// user's configuration. +func pickOriginal(v *verdict) (pick, error) { + p := pick{v: v} + base, err := resolve("HEAD") + if err != nil { + return p, err + } + for _, c := range v.commits { + status, files, err := applyPick("-c", "core.hooksPath=/dev/null", "cherry-pick", "-x", "--signoff", "--end-of-options", c) + if err != nil { + if rerr := resetTo(base); rerr != nil { + return p, rerr + } + return p, fmt.Errorf("cherry-pick of %.12s of #%d: %w", c, v.o.number, err) + } + if status == "conflict" { + p.status, p.files, p.failed = "conflict", files, c + return p, resetTo(base) + } + } + head, err := resolve("HEAD") + switch { + case err != nil: + return p, err + case head == base: + p.status = "empty" + return p, nil + case v.o.parents < 2: + p.status, p.commit = "picked", head + return p, nil + } + msgs, err := commitMessages([]string{v.o.merge}) + if err != nil { + return p, err + } + if p.commit, err = commitMerge(base, head, mergeMessage(msgs[v.o.merge].message, v.o.merge)); err != nil { + if rerr := resetTo(base); rerr != nil { + return p, rerr + } + return p, err + } + p.status = "picked" + return p, nil +} + +// mergeMessage is the message of a batch merge: the one of the merge commit +// on main, which says which pull request it was, and a paragraph of its own +// naming that merge commit, so nothing the original's text ends with can pass +// for the trailer. +func mergeMessage(message, merge string) string { + return strings.TrimRight(message, " \t\r\n") + "\n\n" + batchTrailer + ": " + merge + "\n" +} + +// commitMerge makes the merge of tip onto base, whose tree is tip's since tip +// descends from base, and moves the current branch from tip to it. +func commitMerge(base, tip, message string) (string, error) { + f, err := os.CreateTemp("", "backport-batch-merge-*.txt") + if err != nil { + return "", err + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(message); err != nil { + f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + out, err := git("commit-tree", "-p", base, "-p", tip, "-F", f.Name(), tip+"^{tree}") + if err != nil { + return "", err + } + merge := strings.TrimSpace(out) + if err := checkSHA(merge); err != nil { + return "", fmt.Errorf("commit-tree: %w", err) + } + if _, err := git("update-ref", "-m", "backport-batch: merge", "HEAD", merge, tip); err != nil { + return "", err + } + return merge, nil +} + +// resetTo takes the current branch, its index and its tree back to commit, +// which run owns: it starts only in a clean tree. +func resetTo(commit string) error { + _, err := git("reset", "--quiet", "--hard", commit) + return err +} + +// unit is one original as a batch carries it: a commit on the batch's +// first-parent line, and for a merge the picks its second parent brings. +type unit struct { + commit string + parents []string + message string + // picks are the commits reachable from the second parent and not from the + // first, oldest first. + picks []string +} + +func (u unit) merge() bool { return len(u.parents) == 2 } + +func readUnit(commit string) (unit, error) { + msgs, err := commitMessages([]string{commit}) + if err != nil { + return unit{}, err + } + u := unit{commit: commit, parents: msgs[commit].parents, message: msgs[commit].message} + if u.merge() { + if u.picks, err = revList("--reverse", "--topo-order", "--end-of-options", u.parents[1], "^"+u.parents[0]); err != nil { + return unit{}, err + } + } + return u, nil +} + +// batchUnits lists the units of the batch from base to head, oldest first. +func batchUnits(head, base string) ([]unit, error) { + oids, err := revList("--reverse", "--first-parent", "--end-of-options", head, "^"+base) + if err != nil { + return nil, err + } + var units []unit + for _, oid := range oids { + u, err := readUnit(oid) + if err != nil { + return nil, err + } + units = append(units, u) + } + return units, nil +} + +// unitOriginal is the original a unit says it carries, or nil: the one its +// Backported-from trailer names, or the one its appended -x line picks. +func (b *batch) unitOriginal(u unit) *verdict { + if u.merge() { + if merges := batchedFrom(u.message); len(merges) == 1 { + return b.findMerge(merges[0]) + } + return nil + } + source := appendedSource(u.message) + if v := b.findMerge(source); v != nil { + return v + } + return b.pickedBy([]string{source}) +} + +// ownerOf is the original one of whose own commits source is, or nil. It reads +// every original's commits, so it is for explaining a refusal only. +func (b *batch) ownerOf(source string) (*verdict, error) { + for _, v := range b.verdicts { + if v.commits == nil { + commits, err := prCommits(v.o) + if err != nil { + return nil, err + } + v.commits = commits + } + if v.o.parents > 1 && slices.Contains(v.commits, source) { + return v, nil + } + } + return nil, nil +} + +// checkUnit says why u is not one original carried in the batch's shape, or +// "" when it is, and which original it carries: a merge whose second parent +// is a straight line from its first, of picks of the pull request's own +// commits in their order, each with the -x line of the commit it came from, +// that names the original's merge commit in its trailer; or for a squash one +// pick of its commit on main. Either changes something, and only files the +// original changes. A pick is known by the -x line git appended, not by +// lines its message brought along, and one that also claims another commit +// of the same original is refused. +func checkUnit(b *batch, u unit) (string, *verdict, error) { + switch len(u.parents) { + case 1: + return checkPick(b, u) + case 2: + return checkMerge(b, u) + } + return fmt.Sprintf("%.12s has %d parents", u.commit, len(u.parents)), nil, nil +} + +func checkPick(b *batch, u unit) (string, *verdict, error) { + source := appendedSource(u.message) + if source == "" { + return fmt.Sprintf("%.12s ends in no (cherry picked from commit …) line; a batch carries originals only", u.commit), nil, nil + } + v := b.findMerge(source) + if v == nil { + owner, err := b.ownerOf(source) + if err != nil { + return "", nil, err + } + if owner != nil { + return fmt.Sprintf("%.12s picks one commit of #%d on its own; a merged pull request goes in as a merge of all its commits", u.commit, owner.o.number), owner, nil + } + return fmt.Sprintf("%.12s picks %s, which merged no pull request into main after %s branched", u.commit, source, b.line.branch()), nil, nil + } + if v.o.parents > 1 { + return fmt.Sprintf("%.12s picks the merge commit of #%d as one commit; a merged pull request goes in as a merge of its own commits", u.commit, v.o.number), v, nil + } + why, err := contained(b, v, u.parents[0], u.commit) + return why, v, err +} + +func checkMerge(b *batch, u unit) (string, *verdict, error) { + merges := batchedFrom(u.message) + if len(merges) != 1 { + return fmt.Sprintf("%.12s is a merge with %d %s trailers, not one", u.commit, len(merges), batchTrailer), nil, nil + } + v := b.findMerge(merges[0]) + switch { + case v == nil: + return fmt.Sprintf("%.12s names %s, which merged no pull request into main after %s branched", u.commit, merges[0], b.line.branch()), nil, nil + case v.o.parents < 2: + return fmt.Sprintf("%.12s merges #%d, which main took as one commit; it goes in as one pick", u.commit, v.o.number), v, nil + case len(u.picks) == 0: + return fmt.Sprintf("%.12s merges nothing", u.commit), v, nil + } + if err := b.load(v); err != nil { + return "", nil, err + } + msgs, err := commitMessages(u.picks) + if err != nil { + return "", nil, err + } + prev, last := u.parents[0], -1 + for _, c := range u.picks { + ps := msgs[c].parents + if len(ps) != 1 || ps[0] != prev { + return fmt.Sprintf("%.12s is not on a straight line of picks from %.12s", c, u.parents[0]), v, nil + } + source := appendedSource(msgs[c].message) + if source == "" { + return fmt.Sprintf("%.12s ends in no (cherry picked from commit …) line", c), v, nil + } + i := slices.Index(v.commits, source) + if i < 0 { + owner, err := b.ownerOf(source) + switch { + case err != nil: + return "", nil, err + case owner != nil: + return fmt.Sprintf("%.12s picks a commit of #%d into the merge of #%d", c, owner.o.number, v.o.number), v, nil + } + return fmt.Sprintf("%.12s picks %s, which is not a commit of #%d", c, source, v.o.number), v, nil + } + if i <= last { + return fmt.Sprintf("%.12s picks %.12s out of the order of #%d's commits", c, source, v.o.number), v, nil + } + for _, claimed := range claimedSources(msgs[c].message) { + if claimed != source && (claimed == v.o.merge || slices.Contains(v.commits, claimed)) { + return fmt.Sprintf("%.12s claims more than one commit of #%d", c, v.o.number), v, nil + } + } + prev, last = c, i + } + why, err := contained(b, v, u.parents[0], u.commit) + return why, v, err +} + +// contained says why the change from base to tip is not inside the files v +// changes, or "" when it is. +func contained(b *batch, v *verdict, base, tip string) (string, error) { + if err := b.load(v); err != nil { + return "", err + } + changes, err := changedFiles(base, tip) + if err != nil { + return "", err + } + if len(changes) == 0 { + return fmt.Sprintf("%.12s changes nothing", tip), nil + } + allowed := paths(v.changes) + var extra []string + for _, p := range paths(changes) { + if !slices.Contains(allowed, p) { + extra = append(extra, strconv.Quote(p)) + } + } + if len(extra) > 0 { + return fmt.Sprintf("%.12s touches %s, which #%d does not touch", tip, strings.Join(extra, ", "), v.o.number), nil + } + return "", nil +} + +// replayUnit picks a unit of the batch again onto HEAD in its own shape: a +// pushed resolution keeps what it resolved. On a conflict it takes the branch +// back to where it was before the unit and names the commit that failed. +func replayUnit(u unit) (string, []string, error) { + base, err := resolve("HEAD") + if err != nil { + return "", nil, err + } + picks := u.picks + if !u.merge() { + picks = []string{u.commit} + } + for _, c := range picks { + status, files, err := applyPick("-c", "core.hooksPath=/dev/null", "cherry-pick", "--end-of-options", c) + if err == nil && status == "conflict" { + return c, files, resetTo(base) + } + if err != nil { + if rerr := resetTo(base); rerr != nil { + return "", nil, rerr + } + return "", nil, fmt.Errorf("picking %.12s again: %w", c, err) + } + } + tip, err := resolve("HEAD") + if err != nil || !u.merge() || tip == base { + return "", nil, err + } + _, err = commitMerge(base, tip, u.message) + return "", nil, err +} + +// summarize says, for each original the batch head carries in units, how many +// commits it brought and which of them differ from the commit they were picked +// from: a resolution, or a pick git had to fit to the line. +func summarize(b *batch, head string, on []*verdict) (map[*verdict]string, error) { + units, err := batchUnits(head, b.lineHead) + if err != nil { + return nil, err + } + out := map[*verdict]string{} + for _, u := range units { + v := b.unitOriginal(u) + if v == nil || !slices.Contains(on, v) { + continue + } + picks := u.picks + if !u.merge() { + picks = []string{u.commit} + } + msgs, err := commitMessages(picks) + if err != nil { + return nil, err + } + var changed []string + for _, c := range picks { + source := appendedSource(msgs[c].message) + if source == "" { + continue + } + same, err := samePatch(source, c) + if err != nil { + return nil, err + } + if !same { + changed = append(changed, codeSpan(c[:12])) + } + } + s := "1 commit" + if len(picks) != 1 { + s = fmt.Sprintf("%d commits", len(picks)) + } + if len(changed) > 0 { + s += ", changed in the pick: " + strings.Join(changed, ", ") + } + out[v] = s + } + return out, nil +} + +func plural(n int, word string) string { + if n == 1 { + return word + } + return word + "s" +} + +// samePatch says whether two commits make the same change to their first +// parents, by git patch-id of their diffs without context, so that a pick +// that only lands among other lines on the line counts as the same. +func samePatch(a, b string) (bool, error) { + pa, err := patchID(a) + if err != nil { + return false, err + } + pb, err := patchID(b) + return pa == pb, err +} + +func patchID(c string) (string, error) { + diff, err := git("diff", "--no-ext-diff", "--no-textconv", "--unified=0", "--end-of-options", c+"^", c) + if err != nil { + return "", err + } + out, err := gitInput(diff, "patch-id", "--stable") + if err != nil { + return "", err + } + id, _, _ := strings.Cut(strings.TrimSpace(out), " ") + return id, nil +} diff --git a/cmd/backport-batch/shape_test.go b/cmd/backport-batch/shape_test.go new file mode 100644 index 000000000..0ff713a7f --- /dev/null +++ b/cmd/backport-batch/shape_test.go @@ -0,0 +1,372 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// A conflict on a commit in the middle of a pull request takes the batch back +// to where it was before that original: none of its commits stays, the +// conflict is recorded with the commit that failed, and the next original is +// still picked. +func TestRunConflictInAMiddleCommit(t *testing.T) { + f := newFixture(t) + f.lines() + f.onLine("release-1.1", "fix(chart): a line-only value", map[string]string{"chart/values.yaml": "replicas: 3\nimage: app:1\n"}) + mc5 := f.prCommits(5, "fix(chart): five", "", + change{files: map[string]string{"app/five.txt": "5\n"}}, + change{files: map[string]string{"chart/values.yaml": "replicas: 5\nimage: app:1\n"}}, + change{files: map[string]string{"app/five.txt": "5, done\n"}}) + own := strings.Fields(f.git("rev-list", "--reverse", mc5+"^1.."+mc5+"^2")) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("the batch carries %q, want only #6", got) + } + if log := f.git("log", "--format=%B", "origin/release-1.1..HEAD"); strings.Contains(log, own[0]) || strings.Contains(log, own[1]) { + t.Errorf("a commit of #5 stayed on the batch:\n%s", log) + } + if out := f.git("status", "--porcelain"); out != "" { + t.Errorf("the tree is left dirty:\n%s", out) + } + if _, err := os.Stat(filepath.Join(f.dir, ".git", "CHERRY_PICK_HEAD")); err == nil { + t.Error("a cherry-pick is left in progress") + } + l, _ := parseLine("release-1.1") + deferred, err := parseDeferred(stdout, l) + if err != nil || deferred[5].Commit != own[1] || !slices.Equal(deferred[5].Files, []string{"chart/values.yaml"}) { + t.Errorf("deferred = %+v, %v; want #5 at %s in chart/values.yaml", deferred, err, own[1]) + } + if !strings.Contains(stdout, "`fix(chart): five` at `"+own[1][:12]+"`: `chart/values.yaml`") { + t.Errorf("the body does not name the commit that failed:\n%s", stdout) + } +} + +// What counts as carrying a merged pull request of several commits: picks +// whose appended -x lines name every commit of its own, as a batch merge's do, +// or a pick whose appended -x line names its merge commit; not some of its +// commits. A Backported-from trailer counts only on a commit from before the +// batch-only day, and only in its trailer block. +func TestPresenceOfAMultiCommitOriginal(t *testing.T) { + cases := []struct { + name string + land func(f *fixture, mc string, own []string) + carried string + }{ + { + name: "a batch merge of its commits", + land: func(f *fixture, mc string, own []string) { + f.git("fetch", "-q", "origin") + base := f.git("rev-parse", "origin/release-1.1") + f.git("switch", "-q", "--detach", base) + for _, c := range own { + f.git("cherry-pick", "-x", c) + } + f.git("switch", "-q", "--detach", f.joined(base, f.git("rev-parse", "HEAD"), "fix(app): five (#5)\n\n"+batchTrailer+": "+mc)) + f.git("push", "-q", "origin", "HEAD:refs/heads/release-1.1") + f.git("switch", "-q", "main") + }, + carried: "its 2 commits picked", + }, + { + name: "a pick whose -x line names the merge commit", + land: func(f *fixture, mc string, _ []string) { + f.onLine("release-1.1", "port\n\n(cherry picked from commit "+mc+")", map[string]string{"app/a.txt": "a\n"}) + }, + carried: "picked in", + }, + { + name: "picks whose -x lines name every commit of its own", + land: func(f *fixture, _ string, own []string) { + f.onLine("release-1.1", "a\n\n(cherry picked from commit "+own[0]+")", map[string]string{"app/a.txt": "a\n"}) + f.onLine("release-1.1", "b\n\n(cherry picked from commit "+own[1]+")", map[string]string{"app/b.txt": "b, half\n"}) + }, + carried: "its 2 commits picked", + }, + { + name: "a pick of one of its two commits", + land: func(f *fixture, _ string, own []string) { + f.onLine("release-1.1", "a\n\n(cherry picked from commit "+own[0]+")", map[string]string{"app/a.txt": "a\n"}) + }, + }, + { + name: "the trailer alone, from before the batch-only day", + land: func(f *fixture, mc string, _ []string) { + f.t.Setenv("GIT_COMMITTER_DATE", "2026-10-01T00:00:00Z") + f.onLine("release-1.1", "fix(app): five (#5)\n\n"+batchTrailer+": "+mc, map[string]string{"app/a.txt": "a\n"}) + }, + carried: "named by a " + batchTrailer + " trailer", + }, + { + name: "the trailer alone, after the batch-only day", + land: func(f *fixture, mc string, _ []string) { + f.t.Setenv("GIT_COMMITTER_DATE", "2026-10-27T00:00:00Z") + f.onLine("release-1.1", "fix(app): five (#5)\n\n"+batchTrailer+": "+mc, map[string]string{"app/a.txt": "a\n"}) + }, + }, + { + name: "the trailer in the body, from before the batch-only day", + land: func(f *fixture, mc string, _ []string) { + f.t.Setenv("GIT_COMMITTER_DATE", "2026-10-01T00:00:00Z") + f.onLine("release-1.1", "port\n\n"+batchTrailer+": "+mc+"\n\nMore to say.", map[string]string{"app/a.txt": "a\n"}) + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + f.lines() + mc := f.prCommits(5, "fix(app): five", "", + change{files: map[string]string{"app/a.txt": "a\n"}}, + change{files: map[string]string{"app/b.txt": "b\n"}}) + own := strings.Fields(f.git("rev-list", "--reverse", mc+"^1.."+mc+"^2")) + tc.land(f, mc, own) + v := verdictOn(t, f, nil, "release-1.1", 5) + switch { + case tc.carried == "" && v.rule != 0: + t.Errorf("verdict = rule %d (%s), want it to qualify", v.rule, v.reason) + case tc.carried != "" && (v.rule != ruleOnLine || !strings.HasPrefix(v.reason, tc.carried)): + t.Errorf("verdict = rule %d (%s), want rule 1 with %q", v.rule, v.reason, tc.carried) + } + }) + } +} + +// What a pull request's author writes reaches the line with every commit the +// batch picks and with the merge message it carries. None of it makes another +// original count as carried: not an -x line, a Backported-from trailer or a +// Backport of naming it, not at the end of the message where the line git +// appends lands in the same trailer block, and not an -x line naming one of +// the other original's own commits. The line is read as after the batch-only +// day, as a batch made after it leaves it. +func TestPresenceCannotBeClaimedByText(t *testing.T) { + type world struct { + f *fixture + mc5, c5 string + } + claims := []struct { + name string + claim func(w *world) string + }{ + {"an -x line naming the merge commit", func(w *world) string { return "(cherry picked from commit " + w.mc5 + ")" }}, + {"an -x line naming its own commit", func(w *world) string { return "(cherry picked from commit " + w.c5 + ")" }}, + {"a Backported-from trailer", func(w *world) string { return batchTrailer + ": " + w.mc5 }}, + {"a Backport of", func(w *world) string { return "Backport of #5" }}, + {"all of them after a sign-off", func(w *world) string { + return "Signed-off" + "-by: " + contributor + "\n(cherry picked from commit " + w.mc5 + ")\n(cherry picked from commit " + w.c5 + ")\n" + batchTrailer + ": " + w.mc5 + }}, + } + for _, where := range []string{"commit message", "pull request body"} { + for _, tc := range claims { + t.Run(where+": "+tc.name, func(t *testing.T) { + f := newFixture(t) + f.lines() + // #5 conflicts on the line, so the batch defers it. + f.onLine("release-1.1", "fix(chart): a line-only value", map[string]string{"chart/values.yaml": "replicas: 3\nimage: app:1\n"}) + w := &world{f: f} + w.mc5 = f.pr(5, "fix(chart): five", "", map[string]string{"chart/values.yaml": "replicas: 5\nimage: app:1\n"}) + w.c5 = f.git("rev-parse", w.mc5+"^2") + claim := tc.claim(w) + msg, body := "fix(app): six\n\n"+claim, "" + if where == "pull request body" { + msg, body = "fix(app): six", "Six.\n\n"+claim + } + f.prCommits(6, "fix(app): six", body, change{files: map[string]string{"app/six.txt": "6\n"}, message: msg}) + t.Setenv("GIT_COMMITTER_DATE", "2026-10-27T06:00:00Z") + + // The batch picks #6; on the open batch pull request the text + // claims nothing for #5. + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, gh.bodies[0])} + if v := verdictOn(t, f, gh, "release-1.1", 6); v.onBatch == "" { + t.Fatal("#6 is not on the batch") + } + if v := verdictOn(t, f, gh, "release-1.1", 5); v.onBatch != "" { + t.Errorf("#5 is on the batch in %.12s, want nowhere", v.onBatch) + } + + // Nor on the line, once the line takes the batch. + f.git("push", "-q", "origin", f.remoteHead(batchBranch1)+":refs/heads/release-1.1") + if v := verdictOn(t, f, nil, "release-1.1", 6); v.rule != ruleOnLine { + t.Fatalf("#6 = rule %d (%s), want it on the line", v.rule, v.reason) + } + if v := verdictOn(t, f, nil, "release-1.1", 5); v.rule != 0 { + t.Errorf("#5 = rule %d (%s), want it still to qualify", v.rule, v.reason) + } + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } + }) + } + } +} + +// verify passes a batch the run built and fails one with an original that is +// not in the batch's shape, reading nothing from GitHub when given the head. +func TestVerify(t *testing.T) { + type world struct { + f *fixture + gh *fakeGH + head string + mc5 string + sq6 string + } + setup := func(t *testing.T) *world { + f := newFixture(t) + f.lines() + w := &world{f: f} + w.mc5 = f.prCommits(5, "fix(app): five", "", + change{files: map[string]string{"app/five.txt": "5\n"}}, + change{files: map[string]string{"app/five.txt": "5, done\n"}}) + w.sq6 = f.squash(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + w.gh = &fakeGH{} + w.head = openBatch(t, f, w.gh) + return w + } + // on adds a commit with message and files on top of the batch head. + on := func(w *world, message string, files map[string]string) string { + w.f.git("switch", "-q", "--detach", w.head) + w.f.apply(files) + w.f.git("commit", "-q", "--allow-empty", "-m", message) + sha := w.f.git("rev-parse", "HEAD") + w.f.git("switch", "-q", "main") + return sha + } + cases := []struct { + name string + head func(w *world) string + refusal string + }{ + {name: "the batch as the run built it", head: func(w *world) string { return w.head }}, + { + name: "an original picked as one commit with -m 1", + head: func(w *world) string { + return on(w, "fix(app): five (#5)\n\n(cherry picked from commit "+w.mc5+")", map[string]string{"app/five.txt": "5, again\n"}) + }, + refusal: "picks the merge commit of #5 as one commit", + }, + { + name: "a commit that is no original", + head: func(w *world) string { + return on(w, "fix(ci): adjust for the line", map[string]string{"ci.txt": "x\n"}) + }, + refusal: "ends in no (cherry picked from commit", + }, + { + name: "an original carried twice", + head: func(w *world) string { + own := strings.Fields(w.f.git("rev-list", "--reverse", w.mc5+"^1.."+w.mc5+"^2")) + p := on(w, "again\n\n(cherry picked from commit "+own[1]+")", map[string]string{"app/five.txt": "5, done again\n"}) + w.f.git("switch", "-q", "--detach", w.head) + w.f.git("merge", "-q", "--no-ff", "-m", "fix(app): five (#5)\n\n"+batchTrailer+": "+w.mc5, p) + sha := w.f.git("rev-parse", "HEAD") + w.f.git("switch", "-q", "main") + return sha + }, + refusal: "carries #5 again", + }, + { + name: "a squash-merged original carried as a merge", + head: func(w *world) string { + p := on(w, "six\n\n(cherry picked from commit "+w.sq6+")", map[string]string{"app/six.txt": "6, again\n"}) + w.f.git("switch", "-q", "--detach", w.head) + w.f.git("merge", "-q", "--no-ff", "-m", "fix(app): six (#6)\n\n"+batchTrailer+": "+w.sq6, p) + sha := w.f.git("rev-parse", "HEAD") + w.f.git("switch", "-q", "main") + return sha + }, + refusal: "which main took as one commit", + }, + { + name: "a merge of the line into the batch", + head: func(w *world) string { + line := w.f.onLine("release-1.1", "fix(chart): on the line", map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}) + w.f.git("switch", "-q", "--detach", w.head) + w.f.git("merge", "-q", "--no-ff", "-m", "Merge branch 'release-1.1' into the batch", line) + sha := w.f.git("rev-parse", "HEAD") + w.f.git("switch", "-q", "main") + return sha + }, + refusal: "with 0 " + batchTrailer + " trailers", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w := setup(t) + head := tc.head(w) + // GitHub is not asked: a failing fake would fail the run. + gh := &fakeGH{err: errString("gh: not to be asked")} + code, stdout, stderr := w.f.exec(gh, "verify", "--line", "release-1.1", "--head", head) + switch { + case tc.refusal == "" && code != 0: + t.Errorf("verify = %d\n%s\n%s", code, stdout, stderr) + case tc.refusal != "" && (code != 1 || !strings.Contains(stdout, tc.refusal)): + t.Errorf("verify = %d, want 1 with %q:\n%s\n%s", code, tc.refusal, stdout, stderr) + } + }) + } + + t.Run("the open batch pull request", func(t *testing.T) { + w := setup(t) + code, stdout, stderr := w.f.exec(w.gh, "verify", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, "3 units at "+w.head[:12]) { + t.Errorf("verify = %d\n%s\n%s", code, stdout, stderr) + } + if code, stdout, _ := w.f.exec(&fakeGH{}, "verify", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, "nothing to verify") { + t.Errorf("verify with no batch open = %d, %q", code, stdout) + } + }) + t.Run("a head that is not here", func(t *testing.T) { + w := setup(t) + if code, _, stderr := w.f.exec(w.gh, "verify", "--line", "release-1.1", "--head", strings.Repeat("ab", 20)); code != 2 || !strings.Contains(stderr, "fetch it first") { + t.Errorf("verify = %d (%s), want 2", code, stderr) + } + }) +} + +// A pick that lands among lines the line changed is the same change, and the +// body does not call it changed; one whose change differs is called so. +func TestSummaryCallsOnlyChangedPicksChanged(t *testing.T) { + f := newFixture(t) + f.release("release-1.0") + f.squash(2, "chore(app): a file to change", "", map[string]string{"app/ctx.txt": "a\nb\nc\nd\ne\n"}) + f.pr(1, "feat(app): something for 1.1", "", map[string]string{"app/feature.txt": "1.1\n"}) + f.release("release-1.1") + f.onLine("release-1.1", "fix(app): a line-only change", map[string]string{"app/ctx.txt": "A\nb\nc\nd\ne\n"}) + f.pr(5, "fix(app): five", "", map[string]string{"app/ctx.txt": "a\nb\nC\nd\ne\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if !strings.Contains(stdout, "`fix(app): five`, 1 commit\n") { + t.Errorf("a pick git only fit to its place is called changed:\n%s", stdout) + } +} diff --git a/cmd/backport-batch/verify.go b/cmd/backport-batch/verify.go new file mode 100644 index 000000000..05023a259 --- /dev/null +++ b/cmd/backport-batch/verify.go @@ -0,0 +1,139 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" +) + +// verify checks every original a batch carries, from the line's head to the +// batch head, against the shape the batch carries it in: what fold checks for +// one resolution, for all of them and for the clean picks too. fold runs only +// where someone resolved a conflict and chose to run it; CI runs verify on +// every push to a batch pull request. +// +// With --head it reads that commit and nothing from GitHub, which is how CI +// runs it: the tool built from main, the batch head fetched as data. +func (a *app) verify(args []string) int { + fs := a.flags("verify") + lineArg := fs.String("line", "", "release line, release-X.Y") + headArg := fs.String("head", "", "the batch head to verify (default: the head of the open batch PR)") + if err := parse(fs, args); err != nil { + return 2 + } + l, err := parseLine(*lineArg) + if err != nil { + return a.fail(err) + } + if *headArg != "" { + if err := checkSHA(*headArg); err != nil { + return a.fail(fmt.Errorf("--head: %w", err)) + } + } + if err := fetch(); err != nil { + return a.fail(err) + } + b, err := shapeOf(l) + if err != nil { + return a.fail(err) + } + head := *headArg + if head == "" { + prs, err := a.gh.openPulls(l.branch()) + if err != nil { + return a.fail(err) + } + pr, warnings, err := batchPull(a.gh, prs, l) + for _, w := range warnings { + a.warn("%s", w) + } + if err != nil { + return a.fail(err) + } + if pr == nil { + fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch()) + return 0 + } + if head, err = batchHead(pr); err != nil { + return a.fail(err) + } + } else if have, err := lookup(head); err != nil { + return a.fail(err) + } else if have == "" { + return a.fail(fmt.Errorf("--head %.12s is not in this repository; fetch it first", head)) + } + units, err := batchUnits(head, b.lineHead) + if err != nil { + return a.fail(err) + } + seen := map[*verdict]string{} + var refused []string + for _, u := range units { + why, v, err := checkUnit(b, u) + if err != nil { + return a.fail(err) + } + if why == "" && seen[v] != "" { + why = fmt.Sprintf("carries #%d again, which %.12s already carries", v.o.number, seen[v]) + } + if why != "" { + refused = append(refused, why) + fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why) + continue + } + seen[v] = u.commit + n := len(u.picks) + if !u.merge() { + n = 1 + } + fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit")) + } + if len(refused) > 0 { + fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head) + return 1 + } + fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head) + return 0 +} + +// shapeOf reads what verify needs of a line, from git alone: its head, and +// every pull request merged into main since it branched, unjudged. +func shapeOf(l line) (*batch, error) { + b := &batch{line: l} + var err error + if b.lineHead, err = resolve("refs/remotes/" + remote + "/" + l.branch()); err != nil { + return nil, err + } + if b.mainHead, err = resolve("refs/remotes/" + remote + "/main"); err != nil { + return nil, err + } + out, err := git("merge-base", "--end-of-options", b.mainHead, b.lineHead) + if err != nil { + return nil, fmt.Errorf("%s shares no history with main: %w", l.branch(), err) + } + b.base = strings.TrimSpace(out) + merges, other, err := mainMerges(b.base, b.mainHead) + if err != nil { + return nil, err + } + b.other = other + for _, o := range merges { + b.verdicts = append(b.verdicts, &verdict{o: o}) + } + return b, nil +} diff --git a/cmd/backport-batch/workflows_test.go b/cmd/backport-batch/workflows_test.go new file mode 100644 index 000000000..97547f6b5 --- /dev/null +++ b/cmd/backport-batch/workflows_test.go @@ -0,0 +1,254 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// workflowWorld has #5 change a workflow and #6 change the app and a file +// under .github/ that is no workflow. +func workflowWorld(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(5, "fix(ci): five", "", map[string]string{".github/workflows/five.yaml": "on: push\n", "app/five.txt": "5\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n", ".github/renovate.json": "{}\n"}) + return f +} + +// With --no-workflow-files, as the scheduled batch runs, an original that +// changes a workflow is left to a maintainer, with its workflow files, and is +// neither a conflict nor deferred with a reason. Without the flag it is picked +// like any other. +func TestRunLeavesWorkflowChangesToAMaintainer(t *testing.T) { + l, _ := parseLine("release-1.1") + f := workflowWorld(t) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files") + if code != 0 { + t.Fatalf("run --no-workflow-files = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q, want only #6", got) + } + deferred, err := parseDeferred(stdout, l) + if e := deferred[5]; err != nil || len(deferred) != 1 || !e.Workflows || e.Reason != "" || e.Commit != "" || !slices.Equal(e.Files, []string{".github/workflows/five.yaml"}) { + t.Errorf("deferred = %+v, %v; want #5 waiting with its workflow file", deferred, err) + } + for _, want := range []string{"### Waiting for a maintainer", "[#5](", "`.github/workflows/five.yaml`", "#changes-to-workflows"} { + if !strings.Contains(stdout, want) { + t.Errorf("the body lacks %q:\n%s", want, stdout) + } + } + if strings.Contains(stdout, "### Deferred") { + t.Errorf("the body lists #5 as deferred:\n%s", stdout) + } + if !strings.Contains(stderr, "waiting for a maintainer: https://github.com/cozystack/cozystack/pull/5 fix(ci): five changes .github/workflows/five.yaml") { + t.Errorf("the run does not say #5 waits:\n%s", stderr) + } + + f = workflowWorld(t) + code, stdout, stderr = f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(ci): five (#5)", "fix(app): six (#6)"}) { + t.Errorf("without the flag picked %q, want #5 and #6", got) + } + if strings.Contains(stdout, "Waiting for a maintainer") { + t.Errorf("without the flag the body has something wait:\n%s", stdout) + } +} + +// Waiting is no way off the line: check lists the original until it is on +// it, the scheduled run that continues the batch keeps it waiting rather than +// trying it as a conflict, notify lists it apart from the conflicts, and a +// maintainer's own run picks it onto the open batch. +func TestWorkflowChangesWaitUntilOnTheLine(t *testing.T) { + f := workflowWorld(t) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check with no batch open = %d, %q; want #5 missing", code, stdout) + } + + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "warning: waiting on #90 for a maintainer, since it changes .github/workflows/: https://github.com/cozystack/cozystack/pull/5 ") { + t.Errorf("check = %d, %q; want 0 and a warning about #5:\n%s", code, stdout, stderr) + } + code, stdout, _ = f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90 for a maintainer") { + t.Errorf("check --cut = %d, %q; want #5 to stop the cut", code, stdout) + } + + code, stdout, stderr = f.exec(gh, "notify", "--line", "release-1.1", "--push") + if code != 0 || !strings.Contains(stdout, "1 originals on #90 change .github/workflows/ and wait for a maintainer") || strings.Contains(stdout, "deferred conflicts") { + t.Errorf("notify = %d, %q; want #5 for a maintainer and no conflict\n%s", code, stdout, stderr) + } + notes := slices.DeleteFunc(slices.Clone(gh.issueComments), func(c issueComment) bool { return !strings.HasPrefix(c.Body, notifyMarker) }) + if len(notes) != 1 || !strings.Contains(notes[0].Body, "[#5](") || !strings.Contains(notes[0].Body, "#changes-to-workflows") || strings.Contains(notes[0].Body, "These conflicts") { + t.Errorf("the comment does not list #5 for a maintainer: %+v", gh.issueComments) + } + + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push") + if code != 0 { + t.Fatalf("run --continue = %d\n%s", code, stderr) + } + l, _ := parseLine("release-1.1") + if deferred, err := parseDeferred(stdout, l); err != nil || !deferred[5].Workflows || len(deferred) != 1 { + t.Errorf("the continued body has %+v, %v; want #5 still waiting", deferred, err) + } + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): six (#6)", "fix(app): seven (#7)"}) { + t.Errorf("the continued batch carries %q", got) + } + + f.git("switch", "-q", "main") + reopen(f, gh) + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 { + t.Fatalf("a maintainer's run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", f.remoteHead(batchBranch1)); !slices.Equal(got, []string{"fix(app): six (#6)", "fix(app): seven (#7)", "fix(ci): five (#5)"}) { + t.Errorf("after a maintainer's run the batch carries %q, want #5 on top", got) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || strings.Contains(stderr, "waiting") { + t.Errorf("check after the maintainer's run = %d, %q\n%s", code, stdout, stderr) + } +} + +// --defer keeps what waits as it is, and takes one out like any other original +// when a maintainer gives the reason. +func TestDeferKeepsWorkflowChangesWaiting(t *testing.T) { + f := workflowWorld(t) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + l, _ := parseLine("release-1.1") + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "7", "--reason", "fails the app e2e", "--push") + if code != 0 { + t.Fatalf("run --defer 7 = %d\n%s", code, stderr) + } + if deferred, err := parseDeferred(stdout, l); err != nil || !deferred[5].Workflows || deferred[7].Reason != "fails the app e2e" || len(deferred) != 2 { + t.Errorf("after --defer 7 the body has %+v, %v; want #5 waiting and #7 deferred", deferred, err) + } + + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "waits for the 1.1.4 workflow rework", "--push") + if code != 0 { + t.Fatalf("run --defer 5 = %d\n%s", code, stderr) + } + if deferred, err := parseDeferred(stdout, l); err != nil || deferred[5].Workflows || deferred[5].Reason == "" { + t.Errorf("after --defer 5 the body has %+v, %v; want #5 deferred with the reason", deferred, err) + } + f.git("switch", "-q", "main") + reopen(f, gh) + // #6 and #7 are still only on the batch, which fails the cut; #5 does not. + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || strings.Contains(stdout, "#5 changes") || !strings.Contains(stdout, "#6 is only on open batch PR #90") { + t.Errorf("check --cut = %d, %q; want 1 for #6 and #7 alone, a deferred #5 no longer waiting", code, stdout) + } + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ") { + t.Errorf("check = %d, want 0 with a warning about #5:\n%s", code, stderr) + } +} + +// A record that has an original wait and also names a failed commit or a +// reason is refused: it is one or the other. +func TestParseDeferredRefusesAMixedEntry(t *testing.T) { + l, _ := parseLine("release-1.1") + for _, entry := range []string{ + `{"original":5,"files":[],"workflows":true,"reason":"x"}`, + `{"original":5,"files":[],"workflows":true,"commit":"` + strings.Repeat("a", 40) + `"}`, + } { + body := deferredFence + "\n" + `{"line":"release-1.1","deferred":[` + entry + `]}` + "\n```\n" + if _, err := parseDeferred(body, l); err == nil { + t.Errorf("parseDeferred accepted %s", entry) + } + } +} + +// A pull request that changes a workflow in one commit and takes it back in a +// later one changes no workflow in its merge, but its first pick does, and +// the push carries that pick: it waits as well. +func TestRunHoldsAWorkflowChangeTakenBack(t *testing.T) { + f := newFixture(t) + f.lines() + f.prCommits(5, "fix(ci): five", "", + change{files: map[string]string{".github/workflows/five.yaml": "on: push\n"}}, + change{files: map[string]string{".github/workflows/five.yaml": "", "app/five.txt": "5\n"}}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q, want only #6", got) + } + l, _ := parseLine("release-1.1") + if deferred, err := parseDeferred(stdout, l); err != nil || !deferred[5].Workflows || !slices.Equal(deferred[5].Files, []string{".github/workflows/five.yaml"}) { + t.Errorf("deferred = %+v, %v; want #5 waiting on its workflow file", deferred, err) + } +} + +// A rebuild keeps the count of workflow files a waiting original changes +// beyond those the block names. +func TestDeferKeepsTheWaitingFileCount(t *testing.T) { + f := newFixture(t) + f.lines() + files := map[string]string{"app/five.txt": "5\n"} + for i := range filesKept + 2 { + files[fmt.Sprintf(".github/workflows/w%02d.yaml", i)] = "on: push\n" + } + f.pr(5, "fix(ci): five", "", files) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + gh := &fakeGH{} + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "7", "--reason", "fails the app e2e", "--push") + if code != 0 { + t.Fatalf("run --defer = %d\n%s", code, stderr) + } + l, _ := parseLine("release-1.1") + deferred, err := parseDeferred(stdout, l) + if e := deferred[5]; err != nil || !e.Workflows || len(e.Files) != filesKept || e.More != 2 { + t.Errorf("after the rebuild #5 is %d files and %d more, %v; want %d and 2", len(e.Files), e.More, err, filesKept) + } + if want := fmt.Sprintf("%d more", filesKept+2-filesListed); !strings.Contains(stdout, want) { + t.Errorf("the rebuilt body does not say %q:\n%s", want, stdout) + } +} diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go index 625d79820..119ae3007 100644 --- a/cmd/backupstrategy-controller/main.go +++ b/cmd/backupstrategy-controller/main.go @@ -243,6 +243,7 @@ func main() { Namespace: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE"), Name: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAME"), }, + CredentialsCOSISecretName: os.Getenv("BACKUP_CREDENTIALS_COSI_SECRET_NAME"), } if err := gate.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add DefaultObjectsGate runnable") diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index 07e3ca292..e11290aed 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -99,6 +99,7 @@ func main() { var platformSourceName string var platformSourceRef string var platformSourceSecret string + var platformSourceCertSecret string var systemNamespaceMemoryLimit string var systemNamespaceMemoryRequest string @@ -145,6 +146,7 @@ func main() { flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource") flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.") flag.StringVar(&platformSourceSecret, "platform-source-secret", "", "Name of the Secret in cozy-system namespace containing credentials for the platform source. Sets spec.secretRef on the generated OCIRepository or GitRepository. Secret type depends on the source kind: kubernetes.io/dockerconfigjson for OCI (oci://...); Opaque with username+password (or bearerToken) for Git over HTTPS; Opaque with identity (PEM private key) and known_hosts for Git over SSH.") + flag.StringVar(&platformSourceCertSecret, "platform-source-cert-secret", "", "Name of the Secret in cozy-system namespace holding TLS material for an OCI platform source: ca.crt for a registry behind a private CA, and/or tls.crt+tls.key for client certificate auth. Sets spec.certSecretRef on the generated OCIRepository. Not supported for Git sources, where Flux reads ca.crt from the --platform-source-secret Secret.") flag.StringVar(&cozyValuesSecretName, "cozy-values-secret-name", "cozystack-values", "The name of the secret containing cluster-wide configuration values.") flag.StringVar(&cozyValuesSecretNamespace, "cozy-values-secret-namespace", "cozy-system", "The namespace of the secret containing cluster-wide configuration values.") flag.StringVar(&cozyValuesNamespaceSelector, "cozy-values-namespace-selector", "cozystack.io/system=true", "The label selector for namespaces where the cluster-wide configuration values must be replicated.") @@ -189,6 +191,11 @@ func main() { os.Exit(1) } + if err := validatePlatformSourceCertSecret(platformSourceURL, platformSourceCertSecret); err != nil { + setupLog.Error(err, "invalid platform source flags") + os.Exit(1) + } + config := ctrl.GetConfigOrDie() // Create a direct client (without cache) for pre-start operations @@ -278,12 +285,12 @@ func main() { // Generate and install platform source resource if specified if platformSourceURL != "" { - setupLog.Info("Generating platform source resource", "url", platformSourceURL, "name", platformSourceName, "ref", platformSourceRef, "secret", platformSourceSecret) + setupLog.Info("Generating platform source resource", "url", platformSourceURL, "name", platformSourceName, "ref", platformSourceRef, "secret", platformSourceSecret, "certSecret", platformSourceCertSecret) installCtx, installCancel := context.WithTimeout(mgrCtx, 2*time.Minute) defer installCancel() // Use direct client for pre-start operations (cache is not ready yet) - if err := installPlatformSourceResource(installCtx, directClient, platformSourceURL, platformSourceName, platformSourceRef, platformSourceSecret); err != nil { + if err := installPlatformSourceResource(installCtx, directClient, platformSourceURL, platformSourceName, platformSourceRef, platformSourceSecret, platformSourceCertSecret); err != nil { setupLog.Error(err, "failed to install platform source resource") os.Exit(1) } else { @@ -414,7 +421,7 @@ func main() { // installPlatformSourceResource generates and installs a Flux source resource (OCIRepository or GitRepository) // based on the platform source URL -func installPlatformSourceResource(ctx context.Context, k8sClient client.Client, sourceURL, resourceName, refSpec, secretName string) error { +func installPlatformSourceResource(ctx context.Context, k8sClient client.Client, sourceURL, resourceName, refSpec, secretName, certSecretName string) error { logger := log.FromContext(ctx) // Parse the source URL to determine type @@ -432,11 +439,14 @@ func installPlatformSourceResource(ctx context.Context, k8sClient client.Client, var obj client.Object switch sourceType { case "oci": - obj, err = generateOCIRepository(resourceName, repoURL, refMap, secretName) + obj, err = generateOCIRepository(resourceName, repoURL, refMap, secretName, certSecretName) if err != nil { return fmt.Errorf("failed to generate OCIRepository: %w", err) } case "git": + if err := validatePlatformSourceCertSecret(sourceURL, certSecretName); err != nil { + return err + } obj, err = generateGitRepository(resourceName, repoURL, refMap, secretName) if err != nil { return fmt.Errorf("failed to generate GitRepository: %w", err) @@ -532,6 +542,19 @@ func parseSystemNamespaceMemory(rawLimit, rawRequest string) (limit, request res return limit, request, nil } +// validatePlatformSourceCertSecret rejects --platform-source-cert-secret for a +// Git source: GitRepository has no certSecretRef, and Flux reads a Git CA from +// the ca.crt key of the --platform-source-secret Secret instead. +func validatePlatformSourceCertSecret(sourceURL, certSecretName string) error { + if certSecretName == "" { + return nil + } + if sourceType, _, err := parsePlatformSourceURL(sourceURL); err == nil && sourceType == "git" { + return fmt.Errorf("--platform-source-cert-secret is only supported for OCI sources; for a Git source put ca.crt into the --platform-source-secret Secret") + } + return nil +} + // parsePlatformSourceURL parses the source URL and returns the source type and repository URL. // Supports formats: // - oci://registry.example.com/repo @@ -650,7 +673,7 @@ func validateGitRef(refMap map[string]string) error { } // generateOCIRepository creates an OCIRepository resource -func generateOCIRepository(name, repoURL string, refMap map[string]string, secretName string) (*sourcev1.OCIRepository, error) { +func generateOCIRepository(name, repoURL string, refMap map[string]string, secretName, certSecretName string) (*sourcev1.OCIRepository, error) { if err := validateOCIRef(refMap); err != nil { return nil, err } @@ -687,6 +710,12 @@ func generateOCIRepository(name, repoURL string, refMap map[string]string, secre } } + if certSecretName != "" { + obj.Spec.CertSecretRef = &fluxmeta.LocalObjectReference{ + Name: certSecretName, + } + } + return obj, nil } diff --git a/cmd/cozystack-operator/main_test.go b/cmd/cozystack-operator/main_test.go index 6509065f9..136db4603 100644 --- a/cmd/cozystack-operator/main_test.go +++ b/cmd/cozystack-operator/main_test.go @@ -23,6 +23,7 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" "github.com/cozystack/cozystack/internal/marketplace/tapconst" + sourcev1 "github.com/fluxcd/source-controller/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -534,7 +535,7 @@ func TestValidateGitRef(t *testing.T) { func TestGenerateOCIRepository(t *testing.T) { refMap := map[string]string{"tag": "v1.0", "digest": "sha256:abc123"} - obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", refMap, "") + obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", refMap, "", "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -560,10 +561,13 @@ func TestGenerateOCIRepository(t *testing.T) { if obj.Spec.SecretRef != nil { t.Errorf("expected SecretRef to be nil when secretName is empty, got %+v", obj.Spec.SecretRef) } + if obj.Spec.CertSecretRef != nil { + t.Errorf("expected CertSecretRef to be nil when certSecretName is empty, got %+v", obj.Spec.CertSecretRef) + } } func TestGenerateOCIRepository_NoRef(t *testing.T) { - obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{}, "") + obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{}, "", "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -573,14 +577,14 @@ func TestGenerateOCIRepository_NoRef(t *testing.T) { } func TestGenerateOCIRepository_InvalidRef(t *testing.T) { - _, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"branch": "main"}, "") + _, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"branch": "main"}, "", "") if err == nil { t.Fatal("expected error for invalid OCI ref key, got nil") } } func TestGenerateOCIRepository_WithSecret(t *testing.T) { - obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"tag": "v1.0"}, "my-registry-creds") + obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"tag": "v1.0"}, "my-registry-creds", "") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -592,6 +596,147 @@ func TestGenerateOCIRepository_WithSecret(t *testing.T) { } } +func TestGenerateOCIRepository_WithCertSecret(t *testing.T) { + obj, err := generateOCIRepository("my-repo", "oci://registry.example.com/repo", map[string]string{"tag": "v1.0"}, "", "registry-ca") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if obj.Spec.CertSecretRef == nil { + t.Fatal("expected CertSecretRef to be set when certSecretName is non-empty") + } + if obj.Spec.CertSecretRef.Name != "registry-ca" { + t.Errorf("expected CertSecretRef.Name %q, got %q", "registry-ca", obj.Spec.CertSecretRef.Name) + } + if obj.Spec.SecretRef != nil { + t.Errorf("expected SecretRef to stay nil when only certSecretName is set, got %+v", obj.Spec.SecretRef) + } +} + +func newSourceTestClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + scheme := newTestScheme() + if err := sourcev1.AddToScheme(scheme); err != nil { + t.Fatalf("add source scheme: %v", err) + } + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() +} + +// The operator rewrites the whole platform OCIRepository on every start, so a +// certSecretRef patched onto the live object by hand is lost on the next +// restart and source-controller then fails TLS against a registry behind a +// private CA. The CA has to come from the operator's own arguments and survive +// that update path, including a rename of the Secret when the CA is rotated. +func TestInstallPlatformSourceResource_UpdateKeepsCertSecret(t *testing.T) { + existing := &sourcev1.OCIRepository{ + ObjectMeta: metav1.ObjectMeta{Name: "cozystack-platform", Namespace: "cozy-system"}, + Spec: sourcev1.OCIRepositorySpec{URL: "oci://ghcr.io/cozystack/cozystack/cozystack-packages"}, + } + k8sClient := newSourceTestClient(t, existing) + + for _, certSecret := range []string{"registry-ca", "registry-ca-next"} { + if err := installPlatformSourceResource(context.Background(), k8sClient, + "oci://registry.internal:8083/cozystack/cozystack/cozystack-packages", "cozystack-platform", + "digest=sha256:abc123", "registry-creds", certSecret); err != nil { + t.Fatalf("installPlatformSourceResource: %v", err) + } + } + + got := &sourcev1.OCIRepository{} + if err := k8sClient.Get(context.Background(), client.ObjectKey{Namespace: "cozy-system", Name: "cozystack-platform"}, got); err != nil { + t.Fatalf("get OCIRepository: %v", err) + } + if got.Spec.CertSecretRef == nil || got.Spec.CertSecretRef.Name != "registry-ca-next" { + t.Errorf("expected CertSecretRef registry-ca-next after update, got %+v", got.Spec.CertSecretRef) + } + if got.Spec.SecretRef == nil || got.Spec.SecretRef.Name != "registry-creds" { + t.Errorf("expected SecretRef registry-creds after update, got %+v", got.Spec.SecretRef) + } + if got.Spec.URL != "oci://registry.internal:8083/cozystack/cozystack/cozystack-packages" { + t.Errorf("expected URL to be repointed, got %q", got.Spec.URL) + } +} + +// GitRepository has no certSecretRef: Flux reads a Git CA from the ca.crt key +// of the secretRef Secret. Accepting the flag for a Git source would silently +// drop the CA, so it must fail before anything is written. +func TestInstallPlatformSourceResource_GitRejectsCertSecret(t *testing.T) { + k8sClient := newSourceTestClient(t) + + err := installPlatformSourceResource(context.Background(), k8sClient, + "https://git.internal/cozystack/cozystack", "cozystack-platform", "branch=main", "git-creds", "git-ca") + if err == nil { + t.Fatal("expected an error for a cert secret on a Git platform source, got nil") + } + if !strings.Contains(err.Error(), "ca.crt") { + t.Errorf("expected the error to point at the ca.crt key of the source secret, got: %v", err) + } + + list := &sourcev1.GitRepositoryList{} + if err := k8sClient.List(context.Background(), list); err != nil { + t.Fatalf("list GitRepositories: %v", err) + } + if len(list.Items) != 0 { + t.Errorf("expected no GitRepository to be written, got %d", len(list.Items)) + } +} + +// The Git guard runs on every start; one that misfires without a cert secret +// would crash-loop the operator for every Git platform source. +func TestInstallPlatformSourceResource_GitWithoutCertSecret(t *testing.T) { + for _, sourceURL := range []string{ + "https://git.internal/cozystack/cozystack", + "ssh://git@git.internal/cozystack/cozystack", + } { + t.Run(sourceURL, func(t *testing.T) { + k8sClient := newSourceTestClient(t) + + if err := installPlatformSourceResource(context.Background(), k8sClient, + sourceURL, "cozystack-platform", "branch=main", "git-creds", ""); err != nil { + t.Fatalf("installPlatformSourceResource: %v", err) + } + + got := &sourcev1.GitRepository{} + if err := k8sClient.Get(context.Background(), client.ObjectKey{Namespace: "cozy-system", Name: "cozystack-platform"}, got); err != nil { + t.Fatalf("get GitRepository: %v", err) + } + if got.Spec.URL != sourceURL { + t.Errorf("expected URL %q, got %q", sourceURL, got.Spec.URL) + } + if got.Spec.SecretRef == nil || got.Spec.SecretRef.Name != "git-creds" { + t.Errorf("expected SecretRef git-creds, got %+v", got.Spec.SecretRef) + } + }) + } +} + +// Checked right after flag parsing: CRDs and Flux are installed before the +// platform source, so a combination rejected only there would exit after a +// partial install. +func TestValidatePlatformSourceCertSecret(t *testing.T) { + tests := []struct { + name string + sourceURL string + certSecret string + wantErr bool + }{ + {name: "oci with cert secret", sourceURL: "oci://registry.internal:8083/cozystack-packages", certSecret: "registry-ca"}, + {name: "oci without cert secret", sourceURL: "oci://registry.internal:8083/cozystack-packages"}, + {name: "https git without cert secret", sourceURL: "https://git.internal/cozystack/cozystack"}, + {name: "ssh git without cert secret", sourceURL: "ssh://git@git.internal/cozystack/cozystack"}, + {name: "no source url", certSecret: "registry-ca"}, + {name: "https git with cert secret", sourceURL: "https://git.internal/cozystack/cozystack", certSecret: "git-ca", wantErr: true}, + {name: "ssh git with cert secret", sourceURL: "ssh://git@git.internal/cozystack/cozystack", certSecret: "git-ca", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validatePlatformSourceCertSecret(tt.sourceURL, tt.certSecret) + if (err != nil) != tt.wantErr { + t.Errorf("validatePlatformSourceCertSecret(%q, %q) error = %v, wantErr %v", tt.sourceURL, tt.certSecret, err, tt.wantErr) + } + }) + } +} + func TestGenerateGitRepository(t *testing.T) { refMap := map[string]string{"branch": "main", "commit": "abc1234def5678"} obj, err := generateGitRepository("my-repo", "https://github.com/user/repo", refMap, "") diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go new file mode 100644 index 000000000..558908671 --- /dev/null +++ b/cmd/observability-proxy/main.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command observability-proxy serves tenant logs from the VictoriaLogs +// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the +// tenant's namespace scope forced into every upstream request. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "os" + "os/signal" + "sync/atomic" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + + "github.com/cozystack/cozystack/internal/observabilityproxy" +) + +type options struct { + listenAddress string + probeAddress string + clusterDomain string + rootTenant string + kubeconfig string + authCacheTTL time.Duration + requestTimeout time.Duration + tailIdleTimeout time.Duration + shutdownTimeout time.Duration + logs observabilityproxy.LogsLimits + limiter observabilityproxy.LimiterConfig +} + +func parseFlags(args []string) (options, error) { + var o options + fs := flag.NewFlagSet("observability-proxy", flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.StringVar(&o.listenAddress, "listen-address", ":8080", "address of the API listener") + fs.StringVar(&o.probeAddress, "probe-address", ":8081", "address serving /healthz, /readyz and /metrics") + fs.StringVar(&o.clusterDomain, "cluster-domain", "cozy.local", "cluster DNS domain used to reach the stores") + fs.StringVar(&o.rootTenant, "root-tenant", "tenant-root", "tenant namespace hosting the default store") + fs.StringVar(&o.kubeconfig, "kubeconfig", "", "kubeconfig path; in-cluster config when empty") + fs.DurationVar(&o.authCacheTTL, "auth-cache-ttl", 10*time.Second, "how long TokenReview and SubjectAccessReview answers are cached") + fs.DurationVar(&o.requestTimeout, "request-timeout", time.Minute, "deadline for reading the request body, the token and access reviews, and every upstream endpoint but tail; 0 disables it") + fs.DurationVar(&o.tailIdleTimeout, "tail-idle-timeout", 5*time.Minute, "close a tail after this long without a line") + fs.DurationVar(&o.shutdownTimeout, "shutdown-timeout", 15*time.Second, "grace period for in-flight requests on shutdown") + fs.DurationVar(&o.logs.DefaultRange, "logs-default-range", time.Hour, "window used when a logs request has no start") + // One week covers an incident investigation and keeps a single query + // to a fraction of the default one-month log retention. + fs.DurationVar(&o.logs.MaxRange, "logs-max-range", 7*24*time.Hour, "longest [start, end] a logs request may span; wider windows are clamped") + // Prometheus's cap on points per series in a range query. + fs.IntVar(&o.logs.MaxBuckets, "logs-max-buckets", 11000, "most buckets hits and stats_query_range may return over a window") + fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit") + fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set") + fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset") + fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") + fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") + fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") + fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") + fs.IntVar(&o.limiter.MaxTails, "max-tails", 200, "open tails in total; 0 disables") + if err := fs.Parse(args); err != nil { + return o, err + } + // Parsing stops at the first non-flag, so every flag after it would + // be dropped silently. + if fs.NArg() > 0 { + return o, fmt.Errorf("unexpected argument %q", fs.Arg(0)) + } + switch { + case o.rootTenant == "": + return o, errors.New("--root-tenant must not be empty") + case o.clusterDomain == "": + return o, errors.New("--cluster-domain must not be empty") + case o.logs.DefaultRange <= 0 || o.logs.MaxRange < o.logs.DefaultRange: + return o, errors.New("--logs-default-range must be positive and not above --logs-max-range") + // Below four buckets (three steps once one is held back for + // alignment), or a range under three seconds, the step cap that keeps + // vlselect's step alignment inside the range cannot hold. + case o.logs.MaxBuckets < 4: + return o, errors.New("--logs-max-buckets must be at least 4") + case o.logs.MaxRange < 3*time.Second: + return o, errors.New("--logs-max-range must be at least 3s") + case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit: + return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit") + case o.logs.MaxTailResume < 0: + return o, errors.New("--logs-max-tail-resume must not be negative") + case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0: + return o, errors.New("timeouts and the auth cache TTL must not be negative") + // NaN passes every comparison and +Inf never empties a bucket; either + // would switch the rate limit off without saying so. + case math.IsNaN(o.limiter.RatePerUser) || math.IsInf(o.limiter.RatePerUser, 0): + return o, errors.New("--rate-limit-per-user must be a finite number") + case o.limiter.RatePerUser < 0 || o.limiter.RateBurst < 0 || o.limiter.MaxConcurrentPerUser < 0 || + o.limiter.MaxTailsPerUser < 0 || o.limiter.MaxTails < 0: + return o, errors.New("limits must not be negative; 0 disables a limit") + } + return o, nil +} + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil))) + o, err := parseFlags(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "observability-proxy: %v\n", err) + os.Exit(2) + } + if err := run(o); err != nil { + slog.Error("exiting", "error", err) + os.Exit(1) + } +} + +func run(o options) error { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + restConfig, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) + if err != nil { + return fmt.Errorf("loading kubeconfig: %w", err) + } + client, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("building client: %w", err) + } + + factory := informers.NewSharedInformerFactory(client, 0) + namespaces := factory.Core().V1().Namespaces() + lister := namespaces.Lister() + factory.Start(ctx.Done()) + var ready atomic.Bool + go func() { + if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) { + ready.Store(true) + } + }() + + reg := prometheus.NewRegistry() + reg.MustRegister(collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + proxy := observabilityproxy.NewServer(observabilityproxy.Config{ + ClusterDomain: o.clusterDomain, + Logs: o.logs, + Limiter: o.limiter, + RequestTimeout: o.requestTimeout, + TailIdleTimeout: o.tailIdleTimeout, + }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg) + + probes := http.NewServeMux() + probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) + probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + if !ready.Load() { + http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(w, "ok") + }) + probes.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + + // No WriteTimeout on the API listener: it would cut live tails, which + // the handler bounds with its own idle timeout instead. + servers := []*http.Server{ + {Addr: o.listenAddress, Handler: proxy, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute}, + {Addr: o.probeAddress, Handler: probes, ReadHeaderTimeout: 10 * time.Second}, + } + errs := make(chan error, len(servers)) + for _, srv := range servers { + go func() { + slog.Info("listening", "address", srv.Addr) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + errs <- err + } + }() + } + + var runErr error + select { + case <-ctx.Done(): + case runErr = <-errs: + } + stop() + shutdownCtx, cancel := context.WithTimeout(context.Background(), o.shutdownTimeout) + defer cancel() + for _, srv := range servers { + if err := srv.Shutdown(shutdownCtx); err != nil { + // Open tails never go idle on their own; cut them so clients + // reconnect with since to another replica. + _ = srv.Close() + } + } + return runErr +} diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go new file mode 100644 index 000000000..ea678461c --- /dev/null +++ b/cmd/observability-proxy/main_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + "time" +) + +func TestParseFlagsDefaults(t *testing.T) { + o, err := parseFlags(nil) + if err != nil { + t.Fatal(err) + } + if o.listenAddress != ":8080" || o.probeAddress != ":8081" || o.clusterDomain != "cozy.local" || o.rootTenant != "tenant-root" { + t.Errorf("contract defaults changed: %+v", o) + } + if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour { + t.Errorf("limit defaults changed: %+v", o) + } +} + +func TestParseFlagsOverrides(t *testing.T) { + o, err := parseFlags([]string{"--cluster-domain=example.org", "--logs-max-range=24h", "--max-tails=3"}) + if err != nil { + t.Fatal(err) + } + if o.clusterDomain != "example.org" || o.logs.MaxRange != 24*time.Hour || o.limiter.MaxTails != 3 { + t.Errorf("overrides not applied: %+v", o) + } + // The smallest bucket count and range the step cap can satisfy. + if _, err := parseFlags([]string{"--logs-max-buckets=4", "--logs-max-range=3s", "--logs-default-range=1s"}); err != nil { + t.Errorf("smallest satisfiable step cap refused: %v", err) + } +} + +func TestParseFlagsInvalid(t *testing.T) { + for _, args := range [][]string{ + {"--root-tenant="}, + {"--cluster-domain="}, + {"--logs-default-range=48h", "--logs-max-range=24h"}, + {"--logs-max-buckets=3"}, + {"--logs-max-range=2s", "--logs-default-range=1s"}, + {"--shutdown-timeout=-1s"}, + {"--logs-default-limit=0"}, + {"--logs-default-limit=100", "--logs-max-limit=10"}, + {"--logs-max-tail-resume=-1m"}, + {"--request-timeout=-1s"}, + {"--tail-idle-timeout=-1s"}, + {"--auth-cache-ttl=-1s"}, + {"--rate-limit-per-user=-1"}, + {"--rate-limit-per-user=NaN"}, + {"--rate-limit-per-user=+Inf"}, + {"--rate-limit-burst=-1"}, + {"--max-concurrent-per-user=-1"}, + {"--max-tails-per-user=-1"}, + {"--max-tails=-1"}, + {"--unknown"}, + {"unexpected", "--root-tenant=tenant-x"}, + } { + if _, err := parseFlags(args); err == nil { + t.Errorf("parseFlags(%v) accepted", args) + } + } +} diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index f9eeac96b..88b86edbd 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -43,7 +43,7 @@ When the user asks to generate a changelog, follow these steps in the specified - [ ] **MANDATORY**: Extract PR numbers from commit messages, then use `gh pr view` for each PR to get the PR author. Do NOT use commit author. Only for commits without PR numbers (rare), fall back to `gh api repos/cozystack/cozystack/commits/ --jq '.author.login'` - [ ] Step 8: Form new changelog (structure, format, generate contributors list) - [ ] Step 8b: Verify every operator-facing statement against the code at the upper-bound tag (section 8b) -- [ ] Step 9: Verify completeness and save, after running the checks in `docs/release.md` (section 9) +- [ ] Step 9: Verify completeness and save, after running the checks in `docs/release/changelogs.md` (section 9) ### 1. Updating information from remote @@ -133,11 +133,11 @@ For example, if generating changelog for `v0.37.2`: git log v0.37.1..v0.37.2 --pretty=format:"%h - %s (%an, %ar)" ``` -**⚠️ IMPORTANT: Check for backports:** -- Look for commits with "[Backport release-X.Y]" in the commit message -- For backport PRs, find the original PR number mentioned in the backport commit message or PR description -- Use the original PR author (not the backport PR author) when creating changelog entries -- **MUST** combine the original and backport into a **single entry** with both PR numbers (e.g. `#1606, backport #1609`). Do NOT list the original and the backport as two separate entries — that produces duplicates in the changelog. +**⚠️ IMPORTANT: Check for backports.** They reach a release line in two shapes: +- A backport batch ([`docs/release/backports.md`](../release/backports.md)): a merge commit titled `fix(backport): batch for release-X.Y, (#BATCH_PR)`. Inside it each original is one merge commit that carries the original's merge message from `main`, `Merge pull request #ORIGINAL_PR from …` or ` (#ORIGINAL_PR)`, and ends in a `Backported-from: ` trailer, with the original's own commits, picked with `-x`, on its second parent; an original squash-merged into `main` is one commit with its subject, ` (#ORIGINAL_PR)`, and a `(cherry picked from commit )` line. Read the original's number, title and author from those, list the batch with `git log --first-parent` inside it, and never write an entry for the batch PR itself or for a picked commit of an original. +- A single backport PR, from before the batch (the retired label-driven bot) or a release-only fix: commits with "[Backport release-X.Y]" in the message, whose original PR number is in the backport commit message or PR description. +- Use the original PR author (not the backport or batch PR author) when creating changelog entries +- **MUST** combine the original and backport into a **single entry** with both PR numbers (e.g. `#1606, backport #1609`, where a batch's original reads `#1606, backport #BATCH_PR`). Do NOT list the original and the backport as two separate entries — that produces duplicates in the changelog. **For minor releases (vX.Y.0):** Minor releases must include **all changes** from patch releases of the previous minor version. Get commits from the previous minor release tag up to the new tag: @@ -292,7 +292,8 @@ git log v..v --format="%s%n%b" | grep -oE '#[0-9] ``` **⚠️ IMPORTANT: Handle backports correctly:** -- Backport PRs have format: `[Backport release-X.Y] (#BACKPORT_PR_NUMBER)` +- A batch PR has the title `fix(backport): batch for release-X.Y, `. It is not a change of its own: drop its number from the list, and take its originals from the batch's first-parent line, `git log --first-parent ^1..^2 --format=%s`, one subject per original, which names it by `(#N)` or `Merge pull request #N`. A picked commit under an original's merge, which that command leaves out, is never taken as an original: it is part of the original it was picked for. +- Backport PRs from before the batch have format: `[Backport release-X.Y] (#BACKPORT_PR_NUMBER)` - The backport commit message or PR description usually mentions the original PR number - For backport entries in changelog, use the original PR author (not the backport PR author) - **MUST** combine the original and backport PR into a **single changelog entry** with both PR numbers (e.g. `#1606, backport #1609`). NEVER list the original and the backport as two separate entries — that produces visible duplicates and was the primary defect that motivated this rule. @@ -373,7 +374,8 @@ git log v..v --format="%s%n%b" | grep -oE '#[0-9] 4. **For release branches (backports):** - If commit is from `release-X.Y` branch, check if it's a backport - - Find original commit in `main` to get correct PR number: + - An original a batch carries names its merge commit on `main` in the `Backported-from:` trailer of its merge, and each commit picked under that merge names its source in its `(cherry picked from commit …)` line; read those first + - Otherwise find original commit in `main` to get correct PR number: ```bash git log origin/main --grep="" --oneline ``` @@ -615,7 +617,7 @@ Feature Highlights, the breaking-changes section (Migration and Upgrades in a pa ### 9. Verification and saving -**Check the written file.** Once the draft is in `docs/changelogs/v.md`, run the checks in `docs/release.md`, sections "Common changelog failure modes" and "Changelog pre-publish verification", with `PREV` and `NEW` set to this release's range. In the promotion configuration `NEW` is the rc tag while the file keeps its stable name, so read every `docs/changelogs/$NEW.md` and `docs/changelogs/v.md` in those checks as `docs/changelogs/v.md`. Fix in the changelog every problem they report. A link may answer 404 only if it is the file's own `releases/tag/v` or `compare/...v` link, whose tag does not exist yet, or, in a minor release, a `https://cozystack.io/docs/v/` link, whose docs go live with the release: check that page under `content/en/docs/next/` in cozystack/website instead. Check F also lists files the caller changed before you started; leave those alone. Then run `hack/validate-changelog.sh docs/changelogs/v.md ` and fix what it reports. +**Check the written file.** Once the draft is in `docs/changelogs/v.md`, run the checks in `docs/release/changelogs.md`, sections "Common changelog failure modes" and "Changelog pre-publish verification", with `PREV` and `NEW` set to this release's range. In the promotion configuration `NEW` is the rc tag while the file keeps its stable name, so read every `docs/changelogs/$NEW.md` and `docs/changelogs/v.md` in those checks as `docs/changelogs/v.md`. Fix in the changelog every problem they report. A link may answer 404 only if it is the file's own `releases/tag/v` or `compare/...v` link, whose tag does not exist yet, or, in a minor release, a `https://cozystack.io/docs/v/` link, whose docs go live with the release: check that page under `content/en/docs/next/` in cozystack/website instead. Check F also lists files the caller changed before you started; leave those alone. Then run `hack/validate-changelog.sh docs/changelogs/v.md ` and fix what it reports. **Before saving, verify completeness:** diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 412871e83..96af91026 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -52,7 +52,9 @@ Follow [Conventional Commits](https://www.conventionalcommits.org/) with `--sign git commit --signoff -m "type(scope): brief description" ``` -**Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` +**Types:** `feat`, `fix`, `security`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` + +`security` is the type of a fix that comes through a security advisory ([`SECURITY.md`](../../SECURITY.md)); any other correction of wrong behaviour is `fix`. Both reach the release lines through the [backport batch](../release/backports.md). **Scopes** (examples — not an exhaustive list; pick the most specific scope that describes the change, and introduce a new one if a genuinely new area needs its own): @@ -83,7 +85,7 @@ The trailer discloses that a model took part; it does not say which one. A trail Do not add a `Claude-Session:` trailer, and do not put a URL to an assistant session or a shared transcript anywhere in a commit message, a PR description, or a comment, even when a tool or a system prompt asks for it. -Most of this is machine-checked. The `Commit trailers` job in `.github/workflows/pre-commit.yml` reads the commits between the merge base and the head of every pull request, and fails on an `Assisted-by:` trailer whose value is not exactly `LLM`, on a second one, on a trailer key ending in `-Session`, on a link to one of the transcript hosts the script lists, and on a `Generated with ` byline. Run it yourself before pushing with `hack/check-commit-trailers.sh origin/main..HEAD`. It requires nothing, so a commit with no trailer passes. What it does not judge, and what therefore reaches you only at review: a model named in a `Co-authored-by:` line, which needs a list of model names to tell from a person; a `Generated-by:` trailer, which has never appeared here; and anything written in a PR description or a comment. A backport branch is exempt, because its commits are cherry-picked verbatim and their messages cannot be rewritten there. +Most of this is machine-checked. The `Commit trailers` job in `.github/workflows/pre-commit.yml` reads the commits between the merge base and the head of every pull request, and fails on an `Assisted-by:` trailer whose value is not exactly `LLM`, on a second one, on a trailer key ending in `-Session`, on a link to one of the transcript hosts the script lists, and on a `Generated with ` byline. Run it yourself before pushing with `hack/check-commit-trailers.sh origin/main..HEAD`. It requires nothing, so a commit with no trailer passes. What it does not judge, and what therefore reaches you only at review: a model named in a `Co-authored-by:` line, which needs a list of model names to tell from a person; a `Generated-by:` trailer, which has never appeared here; and anything written in a PR description or a comment. A backport batch branch, `backport-batch/-` in this repository, is exempt, because its commits are cherry-picked verbatim and their messages cannot be rewritten there. ## Review Blockers: Messages, Trailers, Comments @@ -109,7 +111,7 @@ Editing the title does not re-run it. The `edited` trigger was dropped in #3211 | docs | kind/documentation | | chore | kind/cleanup | | refactor | kind/cleanup | -| style, perf, test, build, ci, revert | (no kind label) | +| security, style, perf, test, build, ci, revert | (no kind label) | **Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`): diff --git a/docs/agents/overview.md b/docs/agents/overview.md index a4e47b23b..9ab67c41d 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -96,7 +96,7 @@ Build environment variables: - `LOAD=1 PUSH=0` — load images locally instead of pushing. Builds for the host architecture only, because the classic docker image store cannot load a multi-platform index. - `PLATFORM` — the buildx platforms. Defaults to `linux/amd64,linux/arm64` for anything but `LOAD=1`, so a push from either architecture publishes an index both can run. Needs a `docker-container` builder (`BUILDER`) or a docker daemon on the containerd image store, with QEMU registered for the non-native half. Compiling builder stages run natively on `$BUILDPLATFORM` and cross-compile for the target, but every `RUN` step of a stage that runs on the target platform (package installs, `setcap`, install scripts, patch steps) still runs emulated. `packages/core/testing` pins `linux/amd64`, because it carries an x86 KVM sandbox. `packages/core/talos` builds the Talos installer and release assets for the arches in `PLATFORM` with the upstream imager, which needs no QEMU for either; its matchbox image carries the amd64 and arm64 kernel and initramfs whatever `PLATFORM` says, because it network-boots machines of either arch. - `PUSHED_TAGS_LOG` — a file the `image-tags` macro appends every pushed `:` to, one per line. Unset by default; the release build sets it so `hack/stitch-multiarch.sh` can move every tag it pushed, versioned ones included, onto the multi-arch index. The append happens at recipe expansion, so `make -n` writes it too. -- `CACHE_TAG` — the tag of the mode=max build cache each image reads (and writes under `WRITE_CACHE=1`). Defaults to `buildcache`. Apart from the Talos installer and matchbox in the release job, CI builds one architecture per job, and builds arm64 images only for pre-release tags and in a nightly build of main: the images are built again on native arm64 runners under `-arm64` with `CACHE_TAG=buildcache-arm64`, which only the nightly writes. The talos and testing packages get no arm64 leg: the release job builds the Talos installer and matchbox for both arches itself, and testing is amd64 only. For a pre-release, `hack/stitch-multiarch.sh` then joins each pair into one index, `hack/verify-multiarch.sh` fails the rc on any pinned image that is not an amd64+arm64 index, and the packages artifact is republished pinned on the index digests; see `docs/release.md`. The stitch skips any image whose digest is also pinned in a file it does not rewrite, which then fails that gate; the kamaji control-plane provider's gzipped manifest is the one such copy it does rewrite (see `docs/agents/image-refs.md`). +- `CACHE_TAG` — the tag of the mode=max build cache each image reads (and writes under `WRITE_CACHE=1`). Defaults to `buildcache`. Apart from the Talos installer and matchbox in the release job, CI builds one architecture per job, and builds arm64 images only for pre-release tags and in a nightly build of main: the images are built again on native arm64 runners under `-arm64` with `CACHE_TAG=buildcache-arm64`, which only the nightly writes. The talos and testing packages get no arm64 leg: the release job builds the Talos installer and matchbox for both arches itself, and testing is amd64 only. For a pre-release, `hack/stitch-multiarch.sh` then joins each pair into one index, `hack/verify-multiarch.sh` fails the rc on any pinned image that is not an amd64+arm64 index, and the packages artifact is republished pinned on the index digests; see `docs/release/ci-pipeline.md`. The stitch skips any image whose digest is also pinned in a file it does not rewrite, which then fails that gate; the kamaji control-plane provider's gzipped manifest is the one such copy it does rewrite (see `docs/agents/image-refs.md`). ### Values schema generation @@ -150,7 +150,7 @@ Product documentation lives in [`cozystack/website`](https://github.com/cozystac | --- | --- | | `packages///README.md` | Anything scoped to a single package. Prose goes **above** `## Parameters`; that section is `cozyvalues-gen` output and is never hand-edited. The website's `make update-apps` / `update-vms` / `update-networking` / `update-k8s` / `update-services` targets append the whole README onto a published page, but only for the packages named in the `APPS`, `VMS`, `NETWORKING`, `K8S` and `SERVICES` lists of the website `Makefile`, and only from `packages/apps/` and `packages/extra/`: nothing under `packages/system/` is ever imported, and neither is a package that is not on those lists. The sync runs when a release is promoted or tagged, or by hand in the website repository, not when a pull request merges here. Use this route only for a package that is on the lists; for any other the prose reaches no reader, so put it on the website instead. | | `cozystack/website`, under `content/en/docs/` | Operator- and tenant-facing content that spans packages or is task-shaped: guides, how-tos, troubleshooting runbooks, reference tables. Which version directory a page goes into is set by the website's own [`CONTRIBUTING.md`](https://github.com/cozystack/website/blob/main/CONTRIBUTING.md); follow that file rather than a copy of its rules here. | -| `docs/` | Documentation that a chart template, migration, script or workflow references by repo-relative path, plus this repository's own process docs (`docs/agents/`, `docs/changelogs/`, `docs/release.md`). | +| `docs/` | Documentation that a chart template, migration, script or workflow references by repo-relative path, plus this repository's own process docs (`docs/agents/`, `docs/changelogs/`, `docs/release.md` and the pages under `docs/release/`). | | `api///DESIGN.md`, `pkg/apis//DESIGN.md` | Design of an API group, beside the types it describes. | | [`cozystack/community`](https://github.com/cozystack/community) | Design proposals and RFCs. Not this repository. | diff --git a/docs/agents/releasing.md b/docs/agents/releasing.md index dd7742f5e..547e89c04 100644 --- a/docs/agents/releasing.md +++ b/docs/agents/releasing.md @@ -12,7 +12,7 @@ Follow these instructions when the user asks to: ## Instructions -For detailed release process instructions, follow the steps documented in: +For detailed release process instructions, follow the steps documented in the pages under `docs/release/`, starting from their index: **[docs/release.md](../release.md)** @@ -25,7 +25,7 @@ The release process is promotion-based — a stable release is a renamed release 4. Finalize on merge (CI cuts the write-once tag, publishes the release from the merged changelog, and retags the rc images to stable by digest). 5. Merging the parked website docs PR once the release is published. -All detailed steps, workflows, and failure modes are documented in `docs/release.md`. +All detailed steps, workflows, and failure modes are documented in the pages under `docs/release/`, one per topic, and `docs/release.md` links each of them. -For a patch release, or any rc cut from a `release-X.Y` branch, verify the backport state of that branch before tagging: build with `go build ./cmd/backport-audit`, then run `./backport-audit release-X.Y`. It exits non-zero while anything labeled `kind/backport` / `kind/backport-previous` for the line has not landed on it, and prints the URLs. See [`cmd/backport-audit/README.md`](../../cmd/backport-audit/README.md) and [Cherry-pick triage before a patch](../release.md#cherry-pick-triage-before-a-patch). Do not report a branch as ready to cut on the strength of `gh pr list --search "is:merged label:kind/backport"` — that lists what was labeled, not what arrived. +What a patch release carries is decided by the rules in [`docs/release/backports.md`](../release/backports.md), not per pull request: fixes reach the release lines through the backport batch, and nobody labels or opens a backport. For a patch release, or any rc cut from a `release-X.Y` branch, check the branch before tagging: build the batch tool with `go build ./cmd/backport-batch`, then run `./backport-batch check --line release-X.Y --cut`. It exits 1 while a pull request the rules take is not on the line, including one that is only on the open batch pull request, which the cut would not carry, and prints the URLs; what the batch deferred prints as warnings. See [`cmd/backport-batch/README.md`](../../cmd/backport-batch/README.md) and [Cherry-pick triage before a patch](../release/backport-mechanics.md#cherry-pick-triage-before-a-patch). Do not report a branch as ready to cut on the strength of a list of merged pull requests or of the batch pull request's description — they list what merged or was picked, not what arrived. diff --git a/docs/release.md b/docs/release.md index c8620dbe9..175bd37fc 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,6 @@ # Release Workflow -This document is both the process design (what the release model looks like) and the operational playbook (what to actually do, what breaks, what to verify). Companion to [`agents/changelog.md`](./agents/changelog.md), which is the canonical changelog process. +The release documentation is both the process design (what the release model looks like) and the operational playbook (what to actually do, what breaks, what to verify), split into one page per topic under [`release/`](./release/). Companion to [`agents/changelog.md`](./agents/changelog.md), which is the canonical changelog process. ## When to use this guide @@ -15,692 +15,31 @@ You are about to: If you are about to generate a changelog only, jump straight to [`agents/changelog.md`](./agents/changelog.md). -## Introduction - -Cozystack uses a staged release process to ensure stability and flexibility during development. - -There are three types of releases: - -- **Release Candidates (RC)** – Preview versions (e.g., `v1.2.0-rc.1`) used for final testing and validation. -- **Regular Releases** – Final versions (e.g., `v1.2.0`) that are feature-complete and thoroughly tested. -- **Patch Releases** – Bugfix-only updates (e.g., `v1.2.1`) made after a stable release, based on a dedicated release branch. - -All three are matched by `tags.yaml`'s regex `^v\d+\.\d+\.\d+(-(alpha|beta|rc)\.\d+)?$`. The tag push is the trigger for the whole pipeline. **Pre-release tags (`-alpha` / `-beta` / `-rc`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow** (manual dispatch from `main` or `release-X.Y`) — it pushes the tag as the CI app at the dispatch branch's tip, so GitHub fills `base_ref` (`tags.yaml`'s `Get base branch` step refuses tags pushed without a base) and the push triggers `tags.yaml` (a tag pushed with the default `GITHUB_TOKEN` would not). Stable `vX.Y.Z` tags are never pushed by hand at all — they are cut write-once by the promote flow at a PR merge commit. Only `-rc.N` pre-releases feed the promotion flow ([`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) accepts `vX.Y.Z-rc.N` alone); `-alpha.N` / `-beta.N` tags are built and published for download and validated on demand with the [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) button, but are never promoted to a stable release. - -## Release Candidates - -Release candidates are Cozystack versions that introduce new features and are published before a stable release. Their purpose is to help validate stability before finalizing a new feature release. - -Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v1.2.0-rc.1`. The first one is cut from `main`, once all major features for the upcoming release have merged and the release enters its testing phase. - -**Cutting the first rc freezes the line.** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` at the tagged commit, and from that point the release's content is closed: - -- Every later cut for that line — `rc.2`, `rc.3`, an `alpha`/`beta`, or a patch-line rc — must be dispatched from `release-X.Y`. A dispatch from `main` is refused. -- Fixes reach the release only by cherry-pick or backport onto `release-X.Y` (see [Backports](#backports)). -- `main` reopens immediately for the next minor, so feature work never has to wait for the release to ship. - -This is what makes an rc mean something. Previously `release-X.Y` was created only when the stable release merged, so `rc.2` was cut from `main`'s tip and silently absorbed everything that had landed since `rc.1` — the shipped release could contain code that no rc had ever validated. - -**Un-freezing is manual and deliberate.** If an `rc.1` turns out to be unusable and the line has to be re-cut from `main`, delete `release-X.Y` by hand first; the refuse step blocks every other route back to a `main` dispatch, which is the point of it. Nothing automates the deletion, and nothing should — throwing away a frozen line is a decision worth making explicitly. - -Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. After testing, if no critical issues remain, the last good rc is **promoted** to the regular release `vX.Y.0` — the exact rc images retagged by digest, never rebuilt and never tagged from a later commit (see [Regular Releases](#regular-releases)). Promotion cuts the write-once `vX.Y.0` tag at the promote PR's merge commit, fast-forwards the `release-X.Y` maintenance branch the freeze already created, and opens the way for patch releases. - -## Regular Releases - -A regular release `vX.Y.0` is **promoted from a release-candidate that already passed e2e** — never tagged by hand and never rebuilt. The bytes shipped as `vX.Y.0` are the exact `vX.Y.0-rc.N` images, retagged by digest. We'll use `v1.2.0` as an example. - -```mermaid -gitGraph - commit id: "feature" - commit id: "feature 2" - commit id: "feature 3" tag: "v1.2.0-rc.1" -``` - -A regular release sequence runs as follows: - -1. Cut the first release candidate (`v1.2.0-rc.1`) from the last good commit on `main`: run the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `main`**) with the tag. It tags `main`'s HEAD as the CI app, then **creates `release-1.2` at that commit, freezing the line**; `tags.yaml` then builds the rc images, publishes the rc prerelease, pushes the digest-vendored `release-1.2.0-rc.N` staging branch, and runs its mandatory `rc-e2e` job by calling [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) against that published tag. -2. Watch the rc's mandatory full E2E and wait for it to pass, then complete the other checks in [Pre-release verification checklist](#pre-release-verification-checklist). If the latest `tags.yaml` run cannot provide a green `rc-e2e`, a successful manual dispatch of [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) for the same tag is equivalent gate evidence. `main` is already open for the next minor, so anything landing there is **not** in this release: fixes for `1.2` are cherry-picked onto `release-1.2`, and a later rc is cut by dispatching the workflow **from `release-1.2`**. **Optional:** once the notes are worth reviewing, press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.0.md` now and bank it on the `release-1.2.0-rc.N` staging branch — so there is time to review and hand-edit it before promotion. Skipping the button changes nothing; promotion generates the changelog itself. - - ```mermaid - gitGraph - commit id: "feature" - commit id: "feature 2" - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout main - commit id: "feature 4 (next minor)" - checkout release-1.2 - commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" - ``` - -3. Once the checks are green, run the [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) workflow (manual dispatch **from `main`**, the only branch it is dispatched from, whichever line the rc belongs to) with that rc tag. Its `parse` job verifies E2E before any staging: it accepts a successful `rc-e2e` job in the latest `tags.yaml` push run for the tag, or any retained successful manual `e2e-tag.yaml` run whose expanded E2E job name targets the tag. The emergency `skip_e2e_gate` input bypasses this check, logs a loud warning, and marks the promote PR with ⚠️. Before its first registry write, promotion also proves that the selected `main` or `release-X.Y` target already contains the candidate-aware PR gate, finalizer and trusted verifier; backport this pipeline to an existing maintenance branch before promoting its next patch. Promotion **creates no stable name before merge**: dispatch does write to the registry — it publishes one `promotion-*` packages candidate, and that tag is real and publicly listable — but it creates no `vX.Y.Z` tag, moves no `latest`, and publishes no stable installer, so nothing a consumer resolves changes until the PR merges. It: - 1. Rewrites the rc version substring to `1.2.0` and commits that exact tree, serializes its `packages/` directory under a run-specific temporary OCI tag, then adds a second commit pinning the candidate's immutable digest in the installer before pushing `release-1.2.0`. Container digests remain exactly the rc's; the two commits give the OCI revision an honest source SHA without pretending the artifact can contain its own digest. - 2. Drafts the `v1.2.0` release and uploads the (restamped) assets. - 3. Generates `docs/changelogs/v1.2.0.md` (in a parallel job, detached at the rc tag) — or, if you pressed the rc-time `Generate RC Changelog` button, copies the file already banked on the `release-1.2.0-rc.N` staging branch instead of regenerating — and commits it onto the `release-1.2.0` branch, so the changelog is reviewed as part of the promotion and finalize can publish the release with it already in place. Non-blocking: if generation fails the PR still opens, carrying a ⚠️ line that says the changelog is missing. - 4. Opens the `cozystack/website` "update managed apps reference" PR (`update-docs-v1.2.0` branch), generating the docs from the `release-1.2.0` staging branch via `FETCH_REF`. Non-blocking, and **do not merge that PR until `v1.2.0` is published** — merging early breaks the live site's version routing (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)). - 5. Opens the `chore(release): promote v1.2.0-rc.N -> v1.2.0` PR into `release-1.2`, labelled `release` only. The PR body records whether E2E was verified or bypassed, plus ✅/⚠️ changelog and website-docs lines. E2E does not run on this promote PR by default because the immutable rc already passed the mandatory full suite; a maintainer may add `full-e2e` to run it again. The stable retag itself is deferred to the merge (step 5), so an abandoned promotion leaves no stable-named images and cannot wedge a re-promotion; retention protects the temporary candidate for as long as this release PR remains open, then may remove it after 30 days. - - The PR targets the maintenance branch, not `main`: `promote-rc.yaml` prefers `release-X.Y` whenever it exists, and since the rc freeze it always does. The digest-vendored `Prepare release` commit therefore stays on the release line instead of landing on `main`. - - ```mermaid - gitGraph - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout release-1.2 - commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" - branch release-1.2.0 - checkout release-1.2.0 - commit id: "Prepare release v1.2.0" - commit id: "Pin promoted packages artifact" - checkout release-1.2 - merge release-1.2.0 id: "Pull Request" - ``` - -4. Maintainer reviews the PR and the draft release against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr): confirm the E2E-verification line is ✅ (or explicitly accept its ⚠️ bypass), the changelog line is ✅, and note the website-docs line — the `cozystack/website` docs PR that promotion parked, which stays unmerged for now. E2E does not run on this promote PR by default. Then merge — **do not squash-merge**, the stable tag must attach to a real merge commit. GitHub removes the merged branch `release-1.2.0` (the repo has auto-delete-on-merge enabled). -5. CI workflow triggers on merge (this is where every consumer-visible or write-once side effect happens, after the rc E2E gate and PR review): - 1. Pulls the pinned packages candidate by digest and verifies that it matches the merged `packages/` tree apart from the artifact's impossible self-reference, carries no rc image tags, and preserves the original rc artifact's container `repo@digest` set. - 2. Creates the tag `v1.2.0` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. - 3. Cuts the write-once `api/apps/v1alpha1/v1.2.0` Go-module tag at the same commit. - 4. Ensures the `release-1.2` maintenance branch exists at the tag commit. Since the rc freeze created it, this is normally a no-op fast-forward: the merge commit is already its tip. - 5. Publishes the release page (`draft` → `latest`), using `docs/changelogs/v1.2.0.md` from the merge commit as the release body. - 6. Retags the verified packages candidate and rc container images by digest to `v1.2.0` (and `:latest` only when `v1.2.0` is the newest published stable), and publishes the stable `cozy-installer` chart — no container rebuild. -6. Once the release is published, merge the parked `cozystack/website` docs PR (opened back at promote time, step 3.iv) so the live site's version routing flips to `v1.2.0` — its versioned docs and `openapi.json` are on GitHub now. The maintainer can then announce the release to the community. - -```mermaid -gitGraph - commit id: "feature 2" - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout main - commit id: "feature 4 (next minor)" - checkout release-1.2 - commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" - branch release-1.2.0 - checkout release-1.2.0 - commit id: "Prepare release v1.2.0" - checkout release-1.2 - merge release-1.2.0 id: "Release v1.2.0" tag: "v1.2.0" -``` - -Note that `main` never receives the release commit. It forked away at the freeze and carries the next minor's work. - -## Patch Releases - -A patch release now follows exactly the same shape as the regular release above — by the time `vX.Y.0` ships, everything is already happening on `release-X.Y`. The only difference is where the line starts: - -* `release-X.Y` already exists, created by the freeze at `vX.Y.0-rc.1`, so there is no branch to create. -* Patch commits are cherry-picked onto it, and the rc is cut by dispatching **from `release-X.Y`** — the same dispatch used for `rc.2` of the original release. - - -Let's assume that we've released `v1.2.0` and that development is ongoing. -We have introduced a couple of new features and some fixes to features that we have released -in `v1.2.0`. - -Once problems were found and fixed, a patch release is due. - -```mermaid -gitGraph - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout release-1.2 - commit id: "Release v1.2.0" tag: "v1.2.0" - checkout main - commit id: "feature 4" - commit id: "patch 1" - commit id: "feature 5" - commit id: "patch 2" -``` - - -1. The maintainer cherry-picks patch commits from `main` onto the existing `release-1.2` branch. These must be only patches to features that were present in version `v1.2.0`. - - Cherry-picking can be done as soon as each patch is merged into `main`, or directly before the release. - - The `backport` label automates this: [`backport.yaml`](../.github/workflows/backport.yaml) targets the newest existing `release-X.Y` branch, which during a freeze window is the line being stabilised rather than the last published one. - - ```mermaid - gitGraph - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout release-1.2 - commit id: "Release v1.2.0" tag: "v1.2.0" - checkout main - commit id: "feature 4" - commit id: "patch 1" - commit id: "feature 5" - commit id: "patch 2" - checkout release-1.2 - cherry-pick id: "patch 1" - cherry-pick id: "patch 2" - ``` - - When all relevant patch commits are cherry-picked, the branch is ready for release. Build with `go build ./cmd/backport-audit`, then confirm with `./backport-audit release-1.2` — it exits non-zero while anything labeled for the line is still missing from it, and prints the URLs. See [Cherry-pick triage before a patch](#cherry-pick-triage-before-a-patch). - -2. The maintainer cuts a release candidate (`v1.2.1-rc.N`) via the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `release-1.2`**), which tags that branch's `HEAD`. CI builds and publishes the rc, pushes the `release-1.2.1-rc.N` staging branch, and runs the mandatory full `rc-e2e` job against the published tag. -3. Watch `rc-e2e` and wait for it to pass, then complete the other release checks. A successful manual `e2e-tag.yaml` run for `v1.2.1-rc.N` is also valid evidence if the latest `tags.yaml` run does not carry a green `rc-e2e`. **Optional:** press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.1.md` at rc time and bank it on the rc staging branch; promotion then reuses it instead of regenerating. -4. Once the checks are green, run [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) (manual dispatch **from `main`**, as for every promotion, not from `release-1.2`) with that rc tag. It runs the same five jobs as a regular promotion: the dispatch first gates on the latest tag-push `rc-e2e` or a successful tag-correlated manual E2E run (`skip_e2e_gate` overrides it), verifies that `release-1.2` already carries the candidate-aware pipeline, then commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning the candidate on the `release-1.2.1` branch, drafts the `v1.2.1` release with the restamped assets, generates or picks up `docs/changelogs/v1.2.1.md` and commits it onto that branch, opens the parked `cozystack/website` docs PR (updating the existing `v1.2/` directory in place, since a patch does not promote `next/`), and opens the promote PR into `release-1.2` with only the `release` label. The promote PR body carries the same ✅/⚠️ E2E, changelog, and website-docs lines and runs no E2E by default; add `full-e2e` manually to force a repeat. As with a regular release every stable name is deferred to merge; retention protects the temporary candidate while this PR is open and, after it merges, while `release-1.2` still pins it — until finalize retags it stable. - - ```mermaid - gitGraph - commit id: "feature 3" tag: "v1.2.0-rc.1" - branch release-1.2 - checkout release-1.2 - commit id: "Release v1.2.0" tag: "v1.2.0" - checkout main - commit id: "feature 4" - commit id: "patch 1" - commit id: "feature 5" - commit id: "patch 2" - checkout release-1.2 - cherry-pick id: "patch 1" - cherry-pick id: "patch 2" tag: "v1.2.1-rc.1" - branch release-1.2.1 - commit id: "Prepare release v1.2.1" - checkout release-1.2 - merge release-1.2.1 id: "Pull request" - ``` - - Finally, when release is confirmed, the release sequence goes on. - -5. Maintainer reviews the PR against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr), confirms the E2E-verification and changelog lines and notes the parked website-docs line, and merges it — **do not squash-merge**. GitHub removes the merged branch `release-1.2.1` (auto-delete-on-merge is enabled). -6. CI workflow triggers on merge (all consumer-visible or write-once side effects, after the rc E2E gate and PR review): - 1. Verifies the pinned packages candidate against the merged tree and the original rc artifact's container digests. - 2. Creates the tag `v1.2.1` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. - 3. Cuts the write-once `api/apps/v1alpha1/v1.2.1` Go-module tag at the same commit. - 4. Publishes the release page (`draft` → `latest`; `latest` moves only if `v1.2.1` is the newest published stable). - 5. Retags the verified packages candidate and rc container images by digest to `v1.2.1` and publishes the stable `cozy-installer` chart — no container rebuild. -7. Once the release is published, merge the parked `cozystack/website` docs PR, then announce the release to the community. - -## What CI does during the release process - -The numbered process above is implemented by seven workflows, including the reusable/manual [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml), plus the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) that two of them share to produce the changelog. Knowing which job does what makes the failure modes much easier to diagnose. - -1. [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — manual dispatch (from `main` or `release-X.Y`): the sole entry point for creating a pre-release tag. Validates the `-alpha`/`-beta`/`-rc` tag, then pushes it (write-once, at the branch tip) as the CI app so `tags.yaml` fires. Cutting the first `vX.Y.0-rc.N` also creates `release-X.Y`, freezing the line; afterwards a `vX.Y.*` dispatch from `main` is refused. Stable tags are never created here. -2. [`tags.yaml`](../.github/workflows/tags.yaml) — fires on any pre-release tag push: `build-amd64` and `build-arm64` build the pre-release side by side, then `prepare-release` publishes it and pushes its digest-pinned staging branch, then — **only for `-rc.N` tags** — the mandatory `rc-e2e` job calls `e2e-tag.yaml` in the same run (`-alpha.N` / `-beta.N` skip it and are validated on demand with `e2e-tag.yaml`'s manual button). `generate-changelog` and `update-website-docs` also live here but are **backstops only** — both are normally produced at promotion time (see 4); on a stable tag push each self-skips or no-ops when the promote flow already did the work. These stable-tag backstops only fire because finalize pushes the stable tag with the CI app token (a `GITHUB_TOKEN` push triggers no run — see 5). -3. [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) — reusable from `tags.yaml::rc-e2e` and manually dispatchable for a published tag. It checks that the tag's release carries its nocloud disk, resolves the digest-pinned tree and runs the full install, OpenAPI, and Chainsaw suite on Talos containers. A tag cut before the container lane existed has to be dispatched from its own ref. Its dispatch run title is the constant workflow name, so the promote gate correlates a manual run through the successful tag-bearing E2E job name rather than the title. -4. [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) — manual dispatch, from `main` only (the signature over a stable release's assets names that ref, see [Release signatures and SBOM](#release-signatures-and-sbom)). Five jobs: `parse` validates the rc tag and requires green full-E2E evidence before any staging; `promote` rewrites and commits the rc tags, publishes that packages tree as a run-specific temporary reproducible candidate, commits its digest pin in the `release-X.Y.Z` tree, and drafts the stable release; `changelog` calls the reusable `changelog-generate.yaml` **in parallel** to produce `docs/changelogs/vX.Y.Z.md` (regenerating it, or copying an rc-time changelog already committed to the `release-X.Y.Z-rc.N` staging branch by `changelog-rc.yaml` — see 7); `website-docs` (needs `parse`, `promote`) opens the `cozystack/website` docs PR from the staging branch via `FETCH_REF` (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); `open-pr` joins them, commits the changelog onto the staging branch, and opens the `release-X.Y.Z` promote PR with the `release` label but no automatic `full-e2e`. So the changelog and website docs are reviewed as part of the promotion, the already-checked rc suite is not repeated by default, and the PR body carries ✅ verified or ⚠️ bypassed plus ✅/⚠️ changelog and website-docs lines. Both `changelog` and `website-docs` are non-blocking. No stable registry name is created at dispatch. -5. [`.github/workflows/pull-requests.yaml`](../.github/workflows/pull-requests.yaml) verifies a promotion candidate against the prospective merge tree and original rc artifact before the required `E2E Tests` status can turn green. [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) fires after merge and repeats that verification before any stable tag, then cuts the write-once stable + Go-module tags, publishes the release **with the merged changelog as its body**, retags the candidate and rc images to stable by digest (`:latest` gated on newest-stable), and publishes the stable chart. -6. [`update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — fires on pushes to `main` that touch `docs/changelogs/v*.md`; syncs that content into the corresponding GitHub Release body. No longer the primary path (finalize sets the body directly, which is race-free and works for maintenance lines too) — this now covers later edits to a published changelog and manual re-syncs via `workflow_dispatch`. -7. [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) — manual dispatch, **optional**: generate the changelog at rc time instead of waiting for promotion, while there is still time to review it. It calls the same reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) and commits the result to the `release-X.Y.Z-rc.N` staging branch as `docs/changelogs/vX.Y.Z.md`; the promote-time `changelog` job then copies that file instead of running the AI again. Skipping this workflow changes nothing — promote generates the changelog itself, exactly as before. - -### Phase 1 — `build-amd64`, `build-arm64` and `prepare-release` (hard gate) - -On an rc tag push: builds images for both architectures, commits digest pins, creates the draft release, and pushes the `release-X.Y.Z-rc.N` staging branch. `build-amd64` holds the release-exists no-op and the rejection of a hand-pushed stable tag, and builds the amd64 images and the release assets (its `make build` also pushes the packages artifact and the installer chart, pinned on amd64 digests); `prepare-release` stitches, gates, republishes those two from the repinned tree, and publishes the release. (The stable promote PR is opened later by `promote-rc.yaml`, not here. When the stable tag itself is pushed by finalize, a draft already exists, so this job is a no-op — see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize).) - -The commit (`Prepare release vX.Y.Z`, authored by `cozystack-ci[bot]`) is **digest pins and image tags only**: - -- `packages/core/{installer,platform,testing}/values.yaml` -- `packages/system/*/values.yaml` (cozystack-api, dashboard, kamaji, linstor, kubevirt-csi-node, etc.) -- `packages/apps/kubernetes/images/{kubevirt-csi-driver,ubuntu-container-disk-*}.tag` -- `packages/system/dashboard/templates/configmap.yaml` (the `$tenantText` value) - -Things that look surprising in the diff but are normal: - -- **Zero-diff packages**: when buildx fully caches and upstream base images haven't moved, the digest is byte-identical to the previous release and won't appear. Verify the build step actually ran — don't assume "no diff = nothing built." -- **Sudden churn on `ubuntu-container-disk-*` tags**: `cloud-images.ubuntu.com/noble/current/` is a moving target; these often rebuild even without code changes. -- **Switched registry**: if a self-built image moved from `docker.io` to `ghcr.io`, the registry portion of the digest changes — that's a deliberate move, not a regression. -- **`targetVersion` is NOT touched by the release PR.** Platform migration `targetVersion` is bumped earlier in a feature commit (e.g. `[platform] Bump migration targetVersion to 39 for migration 38`). The release PR only re-pins the `platform-migrations` image digest. - -#### Multi-arch images - -Every pre-release is published for amd64 and arm64. `tags.yaml::build-arm64` builds the images on a native arm64 runner and pushes each as `-arm64`, while `build-amd64` builds amd64, the Talos installer and matchbox for both arches, and the assets on x86 at the same time. `build-amd64` hands its stamped tree, the log of the tags it pushed and the assets to `prepare-release` as workflow artifacts; a missing one fails the upload or the download rather than publishing a partial release. `prepare-release` waits for both builds and runs only if both succeeded: when either fails, the rc is not prepared at all, so a release is never published amd64 only. It applies the stamped tree and runs [`hack/stitch-multiarch.sh`](../hack/stitch-multiarch.sh), which joins each image with its arm64 twin into one index, moves onto it every tag the build pushed for that image (the rc tag and, under `PUBLISH_VERSIONED=1`, its component version, which the build records in `PUSHED_TAGS_LOG`), and repins the tree on the index digests, then [`hack/verify-multiarch.sh`](../hack/verify-multiarch.sh), then republishes the packages artifact and the installer chart from the repinned tree. A stable tag skips `build-arm64` and builds nothing: promotion copies the rc's indexes with every platform, so the stable release is exactly as multi-arch as its rc. Because the two builds run in parallel, an rc that is not prepared (its arm64 build failed, or `prepare-release` failed before the stitch) still leaves behind what the amd64 build pushed: amd64-only images under `:` and under the component-version tags, and the `cozystack-packages:` artifact and the installer chart, both pinned on those amd64 digests. A component tag an earlier rc had moved to an index then points at an amd64-only image again, until a run that succeeds stitches it, a re-run of the same run included. - -`verify-multiarch.sh` is a hard gate. It resolves every ref the tree pins, third-party images included, and fails naming each one that is not an index carrying both `linux/amd64` and `linux/arm64`, or that it cannot resolve at all. The fix is to make the image multi-arch (build it for both, or pin a multi-arch digest of the same version); an image that cannot exist for arm64 goes in [`hack/multiarch-allowlist`](../hack/multiarch-allowlist) by repository, with the reason on the same line. - -That gate sees only what the tree pins by digest. An image a vendored chart renders from its own defaults, or one an operator launches at runtime, is invisible to it. `rc-e2e` closes most of that gap: before tearing the sandbox down, [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) reads every image the three Talos nodes pulled from their image stores (`hack/collect-images.sh`), each by the digest the pull resolved to, and runs `verify-multiarch.sh --refs-file` on that list with the same allowlist. A pre-release fails on any single-arch image, on any ref it cannot read or that is still rate-limited after the retries, and on a list the collection could not produce; a stable tag's run only warns, since it ships its rc's images. A tag whose tree predates the audit (it has no `hack/verify-multiarch.sh`) only warns and passes, so an rc cut before the audit existed can still be re-validated. The list and the audit output are in the run's `image-list-` artifact. Neither check covers a package that e2e never installs and whose images the tree does not pin by digest: an arm64 break there shows up only on an arm64 cluster. - -[`build-arm64-nightly.yaml`](../.github/workflows/build-arm64-nightly.yaml) builds `main` for arm64 every day, so an arm64 build break shows up there before the next rc. Its first step prints `verify-multiarch.sh --report` for `main`'s committed tree: `main` pins amd64 images, so its first-party refs are all listed and are expected to be, since the rc's stitch turns them into indexes. A third-party ref that an in-tree rebuild stamps over at build time, as the Harbor components and shadowbox are, is listed too and goes away on the rc. The lines that matter are the third-party refs no build replaces and refs that cannot be resolved; each of those fails the next rc until it is fixed or allowlisted. matchbox is listed too and does not matter: `build-amd64` rebuilds it as a two-arch index. The stitch names the images it skips in its own summary: an image with no `-arm64` twin, one whose digest is also pinned in a file the rewrite does not reach, and a pair that is not one amd64 and one arm64 image. matchbox (already an index) and the e2e sandbox (never built for arm64, and allowlisted) are skipped on every rc and pass the gate; any other skipped image fails it. - -### RC E2E — mandatory check before promotion - -For an rc, `prepare-release` must finish before `tags.yaml::rc-e2e` calls the published-tag lane, so the full suite tests the digest-pinned staging tree that promotion will reuse, and the release it runs against already carries its disk. `promote-rc.yaml::parse` then checks the latest `tags.yaml` push run whose `head_branch` is the rc tag and accepts it only when the reusable E2E job succeeded. GitHub exposes called-workflow job names as `E2E Release Candidate / E2E vX.Y.Z-rc.N (full suite)`, which the gate matches by the exact tag-bearing suffix. - -The alternate evidence path scans retained successful `workflow_dispatch` runs of `e2e-tag.yaml` and accepts one whose direct E2E job is named `E2E vX.Y.Z-rc.N (full suite)`. The workflow's run title is always `E2E Release Tag` and therefore cannot correlate the input tag; artifact names also carry the tag but expire independently, so the job name is the strongest available correlator. If Actions retention or manual deletion removes the run/job metadata, or that job is renamed without updating the matcher, the gate fails closed and tells the maintainer to run the E2E Release Tag button again. `skip_e2e_gate=true` is an emergency override, not alternate evidence: it writes a loud warning and a ⚠️ PR-body line. - -### Phase 2 — the changelog (generated at rc time or promotion, not here) - -The changelog for `vX.Y.Z` is produced by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `changelog` job (a thin call to the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml)) and committed onto the `release-X.Y.Z` staging branch by `open-pr`, so it rides the promote PR and is on the base branch the moment the release publishes. Finalize then uses that file verbatim as the GitHub Release body. Generating it here — on the stable tag push — would be too late by construction: that tag is created by finalize, which has already published the release. It can also be produced earlier, at rc time, via the optional [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) button — which runs the same reusable core and commits the file to the `release-X.Y.Z-rc.N` staging branch, from which the promote-time `changelog` job copies it (validated) instead of regenerating. - -The `changelog` job runs **in parallel** with `promote`, detached at the rc tag (the "promotion" configuration in §2 of [`agents/changelog.md`](./agents/changelog.md)), using a separate **read-only** GitHub App token for the AI step so the model cannot mutate the repo even with `--allow-all-tools`. Every write — the commit onto the staging branch and the PR — happens in `open-pr` under a write-scoped token. - -Generation is deliberately **non-blocking**: a Copilot outage or exhausted token quota must never block a release. Only the AI step carries `continue-on-error`, and `open-pr` is gated on `promote` alone (`if: !cancelled() && needs.promote.result == 'success'`) so the promote PR opens regardless. When the changelog is missing the PR body says so explicitly — that ⚠️ line is the signal to act on. - -Recovery, in order of preference: - -1. Commit `docs/changelogs/vX.Y.Z.md` to the `release-X.Y.Z` branch by hand (follow [`agents/changelog.md`](./agents/changelog.md)). `promote` preserves it across a re-dispatch, and `open-pr` will not overwrite it with generated output. -2. Re-dispatch `promote-rc.yaml` — generation runs again; anything already on the branch wins. -3. Merge without one and let `tags.yaml::generate-changelog` (the backstop) open a changelog PR against `main` after the fact. The release is published with the draft's body until that merges. - -Known failure modes: - -- Copilot quota exhausted (`COPILOT_GITHUB_TOKEN` 402). Refill or rotate, then re-dispatch. -- AI step timeout (hard 30-min cap). -- Output truncated or whitespace-only — caught by the `Verify changelog` step (`hack/validate-changelog.sh`), which accepts either header convention (`# Cozystack vX.Y.Z` for a minor or `# vX.Y.Z ()` for a patch), and requires the leading release-link comment pointing at `releases/tag/vX.Y.Z`, a `compare/...vX.Y.Z` link ending in this version, and at least one `## ` section — deliberately with no line-count floor, since a complete short patch changelog (v1.5.1 ships in 19 lines) must not be rejected as a fragment. It downgrades to the missing-changelog path rather than shipping a fragment as release notes. - -**`tags.yaml::generate-changelog` is the backstop.** It self-skips when the changelog is already on `main`. It earns its keep in two cases: generation failed during promotion, or the promote PR targeted `release-X.Y` so the file never reached `main`. Since the rc freeze the second case is **every** release, minor and patch alike — the promote PR always targets the release line (see [Regular Releases](#regular-releases)), which makes this backstop the only route by which a published changelog reaches `main`. In that case it **ports the reviewed file from the tag commit verbatim** rather than regenerating — a second AI pass there would spend quota and then overwrite already-published, already-reviewed release notes. - -### Phase 3 — `update-website-docs` (now the backstop) - -The `cozystack/website` "update managed apps reference" PR is normally opened **earlier, at promote time**, by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `website-docs` job — not here. That job fetches the README content from the `release-X.Y.Z` staging branch `promote` just pushed (via the website Makefile's `FETCH_REF` variable), so the docs are ready to review alongside the promote PR while the stable tag is still unborn. It refreshes the trunk version pins from the staging ref before promoting `next/` → `vX.Y/`, so a new minor's docs inherit fresh Talos/cozystack pins instead of a stale `next.yaml`. It is **non-blocking**: if it fails (or trips its `FETCH_REF`-support guard against a website checkout that predates that Makefile variable), the promote PR opens with a ⚠️ website-docs line and this tag-time job backfills the same branch. - -**Merge-timing contract:** the website PR must NOT be merged before the release is published (finalize). Merging it early flips the site's latest-version pointer to a version whose versioned docs and `openapi.json` are not on GitHub yet, so every version-pin and API link for the new version 404s. Both the website PR body and the promote PR body say so; park the website PR until the GitHub release for `vX.Y.Z` is live, then merge. - -This tag-time `update-website-docs` job stays as the **backstop**. It runs with `if: !cancelled() && needs.prepare-release.result == 'success'` (surviving a failed/skipped changelog phase, stopping on cancel), decides whether to promote `next/` → `vX.Y/`, runs `make update-all`, stages `content hugo.yaml data/versions`, and force-pushes the same `update-docs-vX.Y.Z` branch. On the normal path — where `promote-rc.yaml::website-docs` already did the work — it no-ops into "No changes to commit". At tag time the stable tag exists, so its `BRANCH=RELEASE_TAG` fetch resolves without needing `FETCH_REF`. For it to fire at all, finalize must push the stable tag with the **CI app token** (see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize)) — a tag pushed with the default `GITHUB_TOKEN` triggers no workflow run. - -If anyone changes the website Makefile to write somewhere else (e.g. `static/`, `i18n/`), the `git add` list must grow in **both** jobs or the PR silently drops files. - -### Phase 4 — review and merge the promote PR - -Reviewer checklist for the `chore(release): promote vX.Y.Z-rc.N -> vX.Y.Z` PR: - -- [ ] The PR body says `✅ RC full e2e was verified`; if it says the gate was bypassed, obtain explicit maintainer acceptance or add `full-e2e` and wait for that opt-in run. -- [ ] `Verify release packages candidate` is green. It pulls the candidate and original rc artifact by digest, compares the candidate to GitHub's prospective merge tree, and feeds the required `E2E Tests` status even when full E2E is not repeated. Do not edit `packages/` on this PR after generation; if the guard reports drift, re-dispatch promotion to create and pin a new run-specific candidate. Note what a green here does *not* cover: GitHub fires no `pull_request` event when the base branch advances, so a `packages/` change merged to the base afterwards is never re-verified on the PR. Finalize repeats the check against the real merge commit and catches that case before any stable name exists — but it catches it after the promote PR has already merged, so a `packages/` change landing on the base while a promote PR is open is worth holding back. -- [ ] The PR body's **website-docs line** is present: if ✅, the `cozystack/website` docs PR is open and **parked** — do NOT merge it until `vX.Y.Z` is published (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); if ⚠️, the tag-time backstop will backfill it after finalize. -- [ ] Diff is digest pins + image tags only — nothing else. -- [ ] No accidental config drift (a value file you don't recognize). -- [ ] If any chart bumped, check the new digest pulls — `crane manifest ghcr.io/cozystack/cozystack/@` should return. -- [ ] Build artifacts on the draft release page exist and match the expected count. -- [ ] The PR includes `docs/changelogs/vX.Y.Z.md`, it has content, and the entries match the commit range (see [Changelog pre-publish verification](#changelog-pre-publish-verification)). If the PR body carries the ⚠️ "No changelog in this PR" line, resolve that before merging — finalize publishes this file as the release body, unless the PR also carries `docs/releasenotes/vX.Y.Z.md` (see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). - -### Phase 5 — `pull-requests-release.yaml` (finalize) - -Fires on merge of a PR that is merged, carries the `release` label, and is authored by `cozystack-ci[bot]` (the last guard closes the "name a branch `release-X.Y.Z`, get it labelled, merge it" hole). Head branch must match `release-X.Y.Z[-suffix]`. This is where **every consumer-visible or write-once side effect of a promotion happens** — after the rc's mandatory E2E was checked at dispatch and the PR merged, making promotion transactional at stable names. Steps: - -1. **Verify the packages candidate before creating any stable tag.** [`hack/verify-promoted-packages.sh`](../hack/verify-promoted-packages.sh) pulls the candidate by the digest committed in installer values, proves it matches the merged `packages/` tree after normalizing only the impossible self-reference, rejects surviving rc image tags, then follows the candidate's embedded pre-push `platformSourceRef` back to the original rc artifact and compares normalized container `repo@digest` sets. Any package edit after candidate publication or any changed container digest aborts finalize while all stable names are still absent. -2. **Create the tag at the merge commit** (write-once). The merge commit of `Prepare release vX.Y.Z` did not exist before the PR opened, so there is nothing to move — the tag is created here for the first time. A pre-existing tag at a different commit fails the step loudly rather than being force-moved (see [Tag immutability](#tag-immutability)). The checkout uses `persist-credentials: false` so this push authenticates as the **CI app** (via `git remote set-url`), not the default `GITHUB_TOKEN` — a `GITHUB_TOKEN` push creates no workflow run, so a persisted-credential push would silently skip `tags.yaml` entirely and its stable-tag backstops (`generate-changelog`, `update-website-docs`) would never fire. -3. **Cut the `api/apps/v1alpha1/vX.Y.Z` Go-module tag** (write-once, stable only) at the same commit, so Go consumers of `api/apps/v1alpha1` get the release. Moved here from `tags.yaml`, whose `prepare-release` body is skipped for a promoted stable (the draft already exists). -4. **Ensure the maintenance branch `release-X.Y` exists** at the tag commit. Since the rc freeze creates it, this is normally a no-op fast-forward — the merge commit is already the branch tip. Still created if missing; updated fast-forward-only — a non-fast-forward update warns and is left for a maintainer rather than being force-updated. -5. **Publish the draft release**, with `docs/releasenotes/vX.Y.Z.md`, or without it `docs/changelogs/vX.Y.Z.md`, from the merge commit as its body (absent or whitespace-only files warn and leave the draft's body in place, rather than publishing blank notes, and a body over 100000 bytes is published as a link, see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). `make_latest` is computed against published-non-prerelease tags: prereleases stay `false`; tags older than the current max stay `false` (and the current max is force-restored to `latest` if necessary, so an older patch tag cut after a newer minor won't downgrade `latest`). -6. **Retag the verified packages candidate and rc container images to stable** by digest (`hack/promote-retag.sh`, no container rebuild) and **publish the stable `cozy-installer` chart**. `:latest` (on both the images and the chart) moves only when this release's `make_latest` was `true` — the same decision as step 5, so the release's `latest` and the images' `:latest` never disagree. The chart is packaged with `platformVersion` stamped into its default values so the documented `helm --install --version X.Y.Z` path reports the stable version. - -### Phase 6 — `update-releasenotes.yaml` (sync GitHub Release body) - -Fires on pushes to `main` that touch `docs/changelogs/v*.md`. Reads each `docs/changelogs/vX.Y.Z.md` and PATCHes the matching GitHub Release's `body` if it differs. Since Phase 5 now sets the body directly from the merge commit, this is no longer on the critical path — it covers later **edits** to a published changelog, the backstop PR landing a changelog that reached `main` after the fact, and manual re-syncs via `workflow_dispatch`. It only watches `main`, which is why finalize (not this job) is what populates a release promoted onto `release-X.Y` — since the freeze, that is every release rather than only a patch. - -The release body is `docs/releasenotes/vX.Y.Z.md` when it exists and is not blank, otherwise the changelog, which stays the full record either way, and a push to `main` touching either file fires this sync. Finalize sets the body after the stable tags exist, where a body GitHub refuses would strand the release, so at publish time a body over 100000 bytes is replaced by a one-line link to `docs/changelogs/vX.Y.Z.md` on the tag, with a warning in the run's log. This sync never writes that link: it warns and leaves such a release's body as it is. The `tags.yaml` backstop ports only the changelog to `main`, so release notes have to be added to `main` as well, otherwise the sync falls back to the changelog and replaces the release notes with it whenever the changelog fits. - -## Stable tags come from rc promotion - -A stable `vX.Y.Z` is created only by **promoting an existing release-candidate** that already passed e2e — never by rebuilding and never by a cron. - -- [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) is triggered manually once an rc has gone green. It rewrites the rc version substring in the vendored image tags (the container digests stay the rc's), commits and publishes the resulting declarative packages tree under a unique run-specific `promotion-*` tag, commits that digest pin, and opens the `release-X.Y.Z` staging PR. The PR gate verifies the candidate against the prospective merge and original rc artifact; finalize repeats that check before creating the write-once stable tag at the merge commit, then retags the verified packages artifact and rc's already-built, e2e-passed container digests to the stable tag **by digest** (no container rebuild — see [`hack/promote-retag.sh`](../hack/promote-retag.sh)) and publishes the release. An abandoned promotion leaves no stable-named bytes; its temporary candidate is retention-managed. - -So a commit on a supported `release-X.Y` line ships when a maintainer promotes the next rc for that line — not automatically within 24h. Pushing a stable `vX.Y.Z` tag by hand is **not** a supported path: `tags.yaml` fails fast on a stable tag that has no pre-existing draft ("stable tags come from promote-rc.yaml"), and even so the finalize step would refuse to move a pre-existing tag. Pre-releases (`vX.Y.Z-rc.N`, `-alpha.N`, `-beta.N`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow, which pushes the tag as the CI app; [`tags.yaml`](../.github/workflows/tags.yaml) then fires on that push, builds it, and publishes the pre-release. Cutting via the workflow (rather than a manual `git push`) is what lets repo admins lock `v*` tag creation to the CI app — see the tag-protection note below. - -## Release signatures and SBOM - -A release cut from a tree that has [`hack/sign-release-assets.sh`](../hack/sign-release-assets.sh) carries three files beside its other assets, which [`hack/upload-assets.sh`](../hack/upload-assets.sh) has that script write before it uploads anything. Releases cut before it have none of them, and neither does a maintenance line until the script is backported to it (see below). - -- `cozystack-source-sbom.spdx.json` is an SPDX SBOM, made with syft, of the committed source tree (the Go modules, npm lockfiles, GitHub Actions references and other manifests syft reads in it) and of the `cozypkg` binaries in the `cozypkg-*.tar.gz` assets. It does not describe what runs in the cluster: the container images carry their own SBOMs, see [SBOM of the container images](#sbom-of-the-container-images), and the Talos installer image, ISO, disk, kernel and initramfs images have none. Its name says what it covers, so that `cozystack-sbom` stays free for an SBOM of the release as a whole. -- `cozystack-checksums.txt` holds the SHA-256 of every other asset and of the SBOM. -- `cozystack-checksums.txt.sigstore.json` is a Sigstore bundle with a keyless cosign signature over the checksums. Its certificate names the workflow that published the release: `tags.yaml` for a pre-release, `promote-rc.yaml` for a stable one. - -One signature covers every asset through the checksums. To check a download, verify the checksums, then the file against them, with cosign v3.1.3 or later (v2.6.5 or later on the v2 line): - -```sh -cosign verify-blob cozystack-checksums.txt \ - --bundle cozystack-checksums.txt.sigstore.json \ - --certificate-identity-regexp '^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)' \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com && - sha256sum --check --ignore-missing cozystack-checksums.txt -``` - -Older cosign accepts a legacy-format bundle that carries a bare public key in place of a certificate and skips the identity and issuer checks for it ([GHSA-fx35-mq7g-6g98](https://github.com/sigstore/cosign/security/advisories/GHSA-fx35-mq7g-6g98)), so a bundle swapped in next to the download, by a mirror or a proxy, passes there. The bundles this repository writes are in the new format, but cosign picks the format from the file, and `--new-bundle-format` does not stop that on the v3 line (v3.1.2 still accepts such a bundle with it), so the cosign version is what protects the check. - -The regexp pins the ref each workflow runs from: `tags.yaml` on a `v*` tag push, and `promote-rc.yaml` dispatched from `main`, the only branch it is dispatched from. A passing check proves that the checksums were signed by one of those runs in `cozystack/cozystack`. A `promote-rc.yaml` dispatched from another branch fails this check in its own upload step, with no asset uploaded, and an edited `promote-rc.yaml` has to be merged into `main` before it can sign. Until the `v*` [tag rulesets](#repo-side-enforcement-tag-protection-rulesets) restrict tag creation to the CI app, though, anyone with write access can push a `v*` tag whose tree carries an edited `tags.yaml`, and its signature passes. - -The script runs the same `cosign verify-blob` before it returns, so a signature this check would reject fails the job before the first upload, and the release is not published with assets but without a signature. Both jobs that call `make upload_assets`, `tags.yaml::prepare-release` and `promote-rc.yaml::promote`, hold `id-token: write` for the signing identity and install syft and cosign v3.1.3, the version users are asked for above. They install syft from its release tarball, checked against a SHA-256 committed in the workflow, and not with `anchore/sbom-action/download-syft`, which runs an install script fetched at run time in a job that holds the release credentials; [`hack/ci-tool-installs-contract.bats`](../hack/ci-tool-installs-contract.bats) keeps it that way. A stable release gets its own three files rather than its rc's: promotion restamps several assets, so the checksums are recomputed and signed again. - -`promote-rc.yaml` runs the `hack/upload-assets.sh` of the rc tree, not the dispatch ref's. A stable release is therefore signed only when its rc was tagged from a tree that has `hack/sign-release-assets.sh`, and a maintenance line starts carrying signatures once that script is backported to its `release-X.Y` branch. - -### SBOM of the container images - -The container images a release pins by digest each carry an SPDX SBOM of the packages in them, which BuildKit attaches at build time as an attestation (`SBOM=1` in `tags.yaml`, scanner and flags in [`hack/common-envs.mk`](../hack/common-envs.mk)). The attestation is an entry of the image's index, so the digest the release pins names both, and promotion copies it with the image. Read it with `docker buildx imagetools inspect @ --format '{{ json .SBOM }}'`, which prints one document per platform. The attestations are not signed. A release cut before `SBOM=1` was set has none, and neither has the Talos installer image, which the imager builds and `skopeo` pushes, nor the third-party images `packages/` vendors by digest. - -## Nightly builds - -A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs. - -[`nightly.yaml`](../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished: - -1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`. -2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it. -3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply. - -No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package the current nightly publishes, named from the published `cozystack-packages:nightly` tree by [`hack/retention-packages.sh`](../hack/retention-packages.sh), except packages linked to another repository, which are skipped with a warning; the floating `nightly`, release tags and untagged versions are never touched) and abandoned packages candidates. A candidate version is eligible only when all of these hold: it carries at least one tag and **every** tag on that manifest matches the strict `promotion-vX.Y.Z-from-vX.Y.Z-rc.N-run--` form, so a manifest that has been retagged stable is excluded; it was last updated more than 30 days ago; its digest is not pinned by `packages/core/installer/values.yaml` at the head of an **open** pull request that is same-repository, authored by `cozystack-ci[bot]`, on a `release-X.Y.Z` branch and carrying the `release` label; and its digest is not pinned by that same file at the tip of `main` or of any `release-X.Y` branch — which is what keeps a candidate safe between the promote PR merging and finalize retagging it. Any such PR head whose pin is not exactly one `digest=sha256:<64 hex>` stops the candidate sweep for that run and fails the job, without affecting the nightly sweep. - -Install a nightly: - -```bash -helm upgrade --install cozystack \ - oci://ghcr.io/cozystack/cozystack/cozy-installer --version 0.0.0-nightly.20260626.15992304129.1 -# ...or --version nightly for the latest -``` - -The matching Talos node image is `ghcr.io/cozystack/cozystack/cozystack-nocloud:0.0.0-nightly.20260626.15992304129.1` (pull with `oras`). - -## Backports - -### The backport bot - -[`backport.yaml`](../.github/workflows/backport.yaml) wraps [`korthout/backport-action`](https://github.com/korthout/backport-action) and fires on a merged `main`-targeted PR carrying one of: - -| Label | Target branch | -|-------|---------------| -| `kind/backport` | the newest existing `release-X.Y` branch | -| `kind/backport-previous` | the second-newest existing `release-X.Y` branch | - -Resolution is dynamic at run time, and it reads the branches themselves: the job lists the repository's branches, keeps the ones matching `release-.`, and sorts them numerically descending — so `release-1.10` ranks above `release-1.9`, which a lexicographic sort gets backwards. `backport` takes the first, `backport-previous` the second, so both name a branch that exists by construction; asking for `backport-previous` when only one line exists fails the job rather than inventing a target. Nothing is derived arithmetically — an earlier version computed the previous line as `Y-1`, which named a non-existent branch whenever a minor was skipped. - -**During a freeze window the newest line is the one being stabilised, not the last published one.** `release-X.Y` is created when the first `vX.Y.0-rc.1` is cut, before `vX.Y.0` is published, so from the freeze until the release `backport` targets the release being stabilised and `backport-previous` targets the last published stable. That is the intent: the frozen branch is the only way into the upcoming release, which is when backports matter most. The trade-off is that for the length of the window the line one step further back has no label pointing at it, so a fix that has to reach it before the new release publishes must be cherry-picked by hand. - -The bot creates a backport PR with title `[Backport release-X.Y] `. When this PR merges, the title prefix used to re-trigger the bot through `pr-labeler.yaml`, which auto-applied `backport` to any `[Backport release-X.Y]`-titled PR. Combined with the org-level `dosubot` re-applying the label, this caused recursive backports. - -The fix (PR #2584): both job `if:` blocks gate on `github.event.pull_request.base.ref == 'main'`. Backport PRs target `release-X.Y`, so they cannot satisfy this — architectural protection, regardless of which bot relabels them. - -### When the bot fails - -If the run triggered by the merge dies without opening a backport PR, look for a run of this workflow that has not finished before retrying — `gh run list --workflow backport.yaml --branch ` — and clear it, because the retry queues behind it rather than replacing it. Do not filter by status: a run whose jobs are waiting for a runner, or which is held behind the concurrency group, is not `in_progress`, and filtering it out is how you conclude nothing is running. `--branch` narrows the list but does not identify the PR, since head branch names are not unique across forks and `gh run list` has no PR filter, so confirm which PR a run belongs to before cancelling it or you cancel someone else's backport. If cancelling does not clear the run it holds the group until GitHub retires it and the retry keeps queueing; that case has not been exercised here. A run whose jobs have all finished can still sit `in_progress`, which looks the same from the PR as a run that died. Then remove whichever backport label the PR already carries and add it back: a label event is the only other trigger, and it qualifies on the label it carries, so re-applying it is what starts a fresh attempt. - -Conflicting cherry-picks produce a draft PR via `conflict_resolution: draft_commit_conflicts`. Look for the bot's comment with the merge-conflict diff. You either: - -- Resolve in the draft branch and undraft, or -- Drop the bot's branch and cherry-pick manually: - -```bash -git checkout release-X.Y -git cherry-pick -x -s -# resolve conflicts -git commit -s -git push origin release-X.Y # or push to a new branch and open a PR -``` - -To find the bot's failed comments across a batch of PRs: - -```bash -for n in $(gh pr list --search "label:kind/backport label:kind/backport-previous merged:>=2026-01-01" --json number --jq '.[].number'); do - echo "=== #$n ===" - gh pr view $n --json comments --jq '.comments[] | select(.author.login == "github-actions" or (.author.login | contains("backport"))) | .body' | head -20 -done -``` - -Before [PR #3155](https://github.com/cozystack/cozystack/pull/3155), `conflict_resolution` was passed outside `experimental`, so a conflicting cherry-pick used the `fail` fallback and left only a "Backport failed" comment on the source PR. No draft backport PR was opened. When [`backport-audit`](../cmd/backport-audit/README.md) reports `MISSING`, check those comments and the workflow logs to establish why the change did not arrive. - -### Cherry-pick triage before a patch - -A patch release includes bugfixes for code that shipped in the corresponding minor `vX.Y.0`. Use: - -```bash -# 1. Audit what was labeled for this line and has NOT landed on it. -# Exits non-zero when anything is outstanding; see cmd/backport-audit/README.md. -go build ./cmd/backport-audit -./backport-audit release-X.Y - -# 2. List commits on main since the release branch diverged that are NOT yet on release-X.Y -git merge-base origin/main origin/release-X.Y -git log ..origin/main --grep="(#" --oneline - -# 3. Open PRs that may need labeling before the cut -gh pr list --state open --base main --label kind/bug -``` - -Step 1 covers the currently labeled work: [`backport-audit`](../cmd/backport-audit/README.md) assigns each request to the release line selected at merge time, then checks whether its commits reached the branch. Incomplete delivery, open or dropped backports, missing changes and duplicate open claims fail the gate. Late-added labels can disagree with the bot's target, and branch-opening dates are estimated from rc tags; see the audit's documented limits before deciding what to backport. Steps 2 and 3 cover changes that were never labeled. - -**Include rule:** - -- `kind/bug` — especially destructive (data loss, crash-loop, OOM, eviction). -- `area/security` / CVE. -- Narrow chart fixes whose blast radius is contained (e.g. one operator's resources). -- Patch-line-policy-matching dependency bumps (e.g. cilium 1.X.Y → 1.X.(Y+1) on a `release-A.B` shipping cilium 1.X — never a minor bump, only a patch within the line the branch already ships). - -**Skip rule:** - -- `kind/feature` — even with `lgtm`. A patch release is not a delivery vehicle for features. -- CI-only changes (`area/ci`, test infrastructure). -- Docs-only changes (those land via `update-website-docs` automatically). -- Large multi-package dependency churn (Dependabot Go-deps bumps). -- PRs whose conventional-commit type is `!` (breaking) or that mix `feat` with `fix`. - -**Borderline cases:** - -- Large refactors with `lgtm` AND an obvious bug payload — prefer extracting the bug fix into a narrower commit and backporting only that. -- PRs labeled both `kind/bug` and `kind/feature` — split them. - -### Backporting code that touches monotonic counters - -Migrations are the canonical case. `packages/core/platform/images/migrations/run-migrations.sh` is a linear counter — it runs `seq $CURRENT_VERSION $((TARGET_VERSION - 1))` and stops if `CURRENT >= TARGET`. This creates a forward-upgrade trap. Walking through it with `release-1.2` as the maintenance branch: - -- `main` has `targetVersion=10` and migrations 1–9 on disk. -- The `release-1.2` branch was cut when `targetVersion=5` (migrations 1–4 on disk). -- You backport a new migration to `release-1.2` at slot 5, and naively bump `targetVersion` 5 → 6 to enable it. -- Customer running 1.2.x runs the new migration, stamps `cozystack-version=6` in-cluster. -- On upgrade to 1.3 (main's `targetVersion=10`), the runner walks `seq 6 9` and runs main's migrations 6–9 — **main's own migration at slot 5 never runs.** The backport claimed that slot on `release-1.2`; main's slot 5 holds an unrelated migration that gets silently skipped. - -Mitigations, in order of preference: - -1. **Don't bump `targetVersion` in the backport.** Just ship the migration file. Prior precedent in the repo: an earlier maintenance branch's ACME backport did exactly this — the migration shipped but ran on no clusters until a later commit raised the counter. Pair this with relaxing the chart-side `{{ fail }}` guard on the maintenance branch (chart-side coalesce) so the new value works without the migration having executed. -2. **If you must bump,** add an idempotent duplicate of the now-skipped main migration at a higher slot. Bounded cost (one extra file per affected migration) and survives the skip window. -3. **Audit cross-branch counter state before deciding.** Compare migration numbering across `main`, `release-X.Y`, and `release-X.(Y-1)`. If a backport would create a skip window on any forward-upgrade path, choose option 1. - -This rule generalizes to any monotonic-counter state: schema versions, feature-flag generations, anything that gates "have I run X yet?" with a single integer. - -## Changelogs - -### Where the canonical process lives - -[`agents/changelog.md`](./agents/changelog.md) is the source of truth. Read it end-to-end before generating; do not infer the process from past commits or memory. The CI runs it under Copilot via the reusable `changelog-generate.yaml` — called by `promote-rc.yaml::changelog` at promote time and by the optional `changelog-rc.yaml` at rc time — and, as a backstop, `tags.yaml::generate-changelog`; locally the same prompt is replayable by following the doc directly. - -### Common changelog failure modes - -These are mistakes that have shipped in real changelogs. Verify against each before merging the changelog PR. - -**1. Commits from outside the release range.** Caused by running `git log ..HEAD` from `main` while generating a patch changelog — `HEAD` on `main` contains everything that landed since the tag plus backports merged to `release-X.Y` after the tag was cut. Use `git log ..` explicitly. The doc was updated to require this after v1.3.1's changelog shipped with 14 entries for what was actually a 1-commit release. - -**2. Original + backport listed as two separate entries.** Each backport must coalesce with its original into a single entry of the form `(in #orig, backport #bp)`. To verify after generation: - -```bash -# Every entry that mentions "backport" should also reference the original PR number -grep -E 'backport #[0-9]+' docs/changelogs/v.md | grep -v '#[0-9]*, backport' -``` - -Output should be empty. - -**3. Wrong PR author.** The squash-merge commit's author is whoever clicked "Merge" — not the person who wrote the code. **Always** resolve via `gh pr view --json author --jq .author.login`, never `git log --format=%an`. This bites hardest for website-repo entries where the same merger handles most PRs. - -**4. Superseding patch, stale changelog.** Tags are write-once, so a critical fix after `vX.Y.Z` ships as the next patch `vX.Y.(Z+1)` rather than moving the tag. If a changelog PR already exists for a tag you are superseding, the changelog work transfers to the new patch. Compare `git log ..` against what the existing `changelog-vX.Y.Z` branch already documents; only add/remove the deltas. If the only new commits are CI-internal or a revert of a feature that never reached a stable tag, no changelog edit is needed. - -**5. PR numbers swapped inside prose.** The entry-format validator checks bullet entries but ignores Feature Highlights paragraphs and Upgrade Notes. Both have caused wrong PR references in shipped changelogs. Verify every `#NNNN` in prose with `gh pr view `. - -**6. Hallucinated entries.** Every entry must correspond to a commit in `git log ..`. Verify by extracting PR numbers from the file and grep-searching the commit range: - -```bash -grep -oE '#[0-9]+' docs/changelogs/v.md | sort -u | while read pr; do - n=${pr#\#} - if ! git log .. --grep="#${n}" --oneline | grep -q .; then - echo "PR $pr in changelog but not in commit range" - fi -done -``` - -**7. Bot accounts as human contributors.** `cozystack-ci[bot]`, `github-actions`, `dependabot`, `renovate`, and any `app/*` login must NOT appear in the `## Contributors` list. They legitimately appear in **per-entry attribution** (e.g. a Renovate-authored PR) — but not in the human roll-call. - -**8. Title duplicated as description.** `* **fix(foo): X**: fix(foo): X (...)` means you wrote the conventional-commit subject twice and never wrote a user-facing description. The brief and the detail must say different things — the detail explains what the change means for users. - -### Changelog pre-publish verification - -Run these checks before merging the changelog PR. Catches problems in roughly the order they occur: - -```bash -PREV=v1.2.0; NEW=v1.2.1 - -# A. Every chart bump is mentioned somewhere in the changelog -git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*/*/charts/*/Chart.yaml' | while read f; do - old=$(git show "$PREV:$f" 2>/dev/null | yq '.version // ""') - new=$(git show "$NEW:$f" 2>/dev/null | yq '.version // ""') - if [ "$old" != "$new" ]; then - component=$(basename "$(dirname "$f")") - if ! grep -qi "$component" docs/changelogs/$NEW.md; then - echo "MISSING: $component $old -> $new ($f)" - fi - fi -done - -# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images) -for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do - git diff $PREV..$NEW -- "$path" | head -40 -done -git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile' - -# C. Every cited #NNNN exists and is a merged PR (not an issue) -grep -oE '#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's/#//' | xargs -P 20 -I{} sh -c ' - state=$(gh api /repos/cozystack/cozystack/issues/{} --jq ".pull_request.merged_at // \"ISSUE\"" 2>/dev/null) - echo "#{} $state" -' | grep -E 'ISSUE|null|^#[0-9]+ $' - -# D. URL liveness -grep -oE 'https://[^)" ]+' docs/changelogs/$NEW.md | sort -u | xargs -P 10 -I{} sh -c ' - code=$(curl -s -o /dev/null -w "%{http_code}" {}) - if [ "$code" != "200" ]; then echo "$code {}"; fi -' - -# E. Website PR authors are PR authors, not commit authors -grep -oE 'cozystack/website#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's|cozystack/website#||' | while read n; do - pr_author=$(gh pr view $n --repo cozystack/website --json author --jq .author.login) - echo "website#$n -> @$pr_author" -done - -# F. Working tree clean except for the changelog -git status --porcelain | grep -v "docs/changelogs/$NEW.md" -# Should output nothing. -``` - -Ship criterion: every chart bump surfaced, no issue-cited-as-PR, no 404 URLs, every website entry attributed via `gh pr view` (not commit author), `git status` shows only the changelog. - -### Cozystack-specific URL quirks - -`cozystack.io` paths are versioned: `/docs/operations/foo/` and `/docs/next/foo/` both 404. Only `/docs/vX.Y/foo/` and the unversioned canonical `/docs/foo/` serve content. AI-generated changelogs invent the unversioned form. Verify URLs with the `curl` loop above. - -### Platform-component coverage - -For a minor release (`vX.Y.0`), changelog entries should surface what an upstream chart bump actually brought in — features, breaking changes, security fixes. PR commit messages usually don't capture this; the upstream `CHANGELOG.md` or GitHub Release notes do. For each bumped `Chart.yaml`: - -1. Read the `sources:` field — typically a GitHub URL. -2. Fetch release notes strictly inside `old < x <= new` via `gh release view --repo / --json body,name`. -3. Summarize in 2–5 user-impact bullets per component (new features, breaking changes, CVEs, important deprecations). Don't reproduce upstream notes verbatim; link to them. -4. If a PR-level entry already mentions the bump, enrich that entry with bullets — don't duplicate the entry. -5. If 3+ bumps have no PR-level mention, introduce a `## Platform Components` section. - -If you can't find upstream notes for a bumped component, list it with version-only info and no bullets. Hallucinated bullets are worse than missing detail. - -## Release-time fires — what's gone wrong recently - -These are real regressions that escaped to users. The pattern in each: code passed CI, shipped in a stable release, broke customers. Read them as a working list of pre-release verification gaps. - -### v1.3.0 — cert-manager ingressClassName regression (fix #2562) - -**Bug:** A commit migrated ACME HTTP-01 to use the modern `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The matching ingress-shim code that reads that annotation was added by cert-manager upstream PR #8244 and was **never backported** to the `release-1.19` branch — but cozystack 1.3.0 shipped cert-manager v1.19.3. The annotation was silently dropped at runtime and every solver Ingress fell back to the ClusterIssuer's default class `tenant-root`. Let's Encrypt validation broke for every tenant whose `publishing.ingressName != tenant-root`. - -**Blast radius:** v1.3.0, v1.3.1, and v1.1.7 (the migration was cherry-picked to `release-1.1`). - -**Why CI missed it:** the migration commit was self-contained and `helm template` rendered cleanly. There was no E2E that creates a tenant with a non-default ingress class and watches a real Let's Encrypt cert issue. Verification had been done from the docs side ("the API has been available since cert-manager 1.12"), not against the shipped cert-manager source. - -**Would have caught it:** non-root-tenant E2E with LE-staging cert issuance, asserting the solver Ingress lands on the tenant's class. - -### v1.4.0 — kubevirt-instancetypes upgrade failure (fix #2612) - -**Bug:** `packages/system/kubevirt-instancetypes/Makefile` contained `sed -i '/persistent: true/d' templates/preferences.yaml`, which stripped the only child of `preferredTPM`, leaving `preferredTPM:` (null) in six Windows preferences. KubeVirt v1.6.x silently accepted it. The v1.6.3 → v1.8.2 operator bump (2026-04-27) hardened the CRD's OpenAPI schema and started rejecting null for an object-typed field. `helm upgrade` then failed on every cluster with `kubevirt-instancetypes` enabled. - -**Why CI missed it:** the kubevirt-operator bump and the inert `preferences.yaml` were two weeks apart in commit history. CI only tests fresh installs against the new CRDs; it doesn't run `helm upgrade` from prior versions. Schema regression on a vendored asset that hadn't been touched in 18 months. - -**Would have caught it:** `helm template | kubectl apply --dry-run=server -f -` against the freshly-installed CRDs of every bumped operator. Or an upgrade-from-N-1 E2E lane. - -### v1.4.0 — Flux 2.7 → 2.8 readiness deadlock (#2602) - -**Bug:** helm-controller v1.5.0 (shipped in Flux 2.8) changes the default wait strategy to kstatus polling, which polls **every applied resource including child HelmReleases**. The `packages/apps/kubernetes` umbrella has 19 child HRs that each `dependsOn: parent` (with a `lookup` guard so the dependency only activates after the parent exists). Pre-2.8: parent went Ready as soon as Helm returned. Post-2.8: parent waits for children, children wait for parent. Deadlock. - -**Why CI missed it:** the cycle only manifests on the second reconcile (parent exists, so the `lookup` activates the child's `dependsOn`). Fresh-install E2E dodges this — the parent-already-exists branch isn't taken on the first apply. - -**Would have caught it:** apply-mutate-reapply E2E (any chart bump in the kubernetes umbrella triggers a second reconcile). Plus an explicit PR-template requirement: for any flux/helm-controller bump, paste the upstream breaking-changes section into the PR body. - -### Patterns these share - -1. **Fresh-install CI is insufficient.** All three regressions passed fresh-install lanes. -2. **Upstream breaking-defaults bumps are the common shape.** kubevirt CRD hardening, helm-controller default wait strategy, cert-manager API surface drift. Pre-merge reading of upstream changelogs would have caught two of three. -3. **One-engineer-bitten-once is the only safety net for live verification.** The cert-manager bump PR included an explicit live-cluster verification step — because the same engineer had been bitten the prior week. There is no checklist requiring it. - -## Pre-release verification checklist - -For RCs and final releases, run this before merging the release PR. Each item is here because something has shipped without it and broken users. - -- [ ] **The published rc passed mandatory full E2E.** Confirm the promote PR says `✅ RC full e2e was verified`; the normal promote PR itself runs no E2E. Add `full-e2e` manually only when a repeat is required, and treat a ⚠️ bypass line as an explicit release-risk decision. -- [ ] **Upgrade from previous patch** (`vX.Y.(Z-1)`) on a real cluster. `helm upgrade` succeeds, all HelmReleases reach Ready, no CRD schema rejections. -- [ ] **Upgrade from previous minor** (`v(X-1).Y.0`) for RC and minor releases. Same assertions. -- [ ] **At least one tenant with `publishing.ingressName != tenant-root`** and a real Let's Encrypt-staging cert issuance. -- [ ] **Apply-mutate-reapply** on the kubernetes umbrella HelmRelease (catches second-reconcile bugs). -- [ ] **For every CRD-source bump** (kubevirt, cert-manager, flux, cilium, kamaji): `helm template | kubectl apply --dry-run=server` against the freshly-installed CRDs. -- [ ] **For every flux / helm-controller / kubevirt-operator / cert-manager bump:** paste the upstream breaking-changes section of the bump's CHANGELOG into the PR body before merge. -- [ ] **After cutting the tag,** watch `pull-requests-release.yaml::Finalize Release`. If it fails on `Draft release for v... not found`, undraft the release manually and file the workflow regression — this has been a known recurring failure since the `Publish draft release` step lost its explicit `github-token` in refactor `66a756b6`. -- [ ] **The promote PR carries `docs/changelogs/vX.Y.Z.md` and it was verified** per [Changelog pre-publish verification](#changelog-pre-publish-verification). Finalize publishes this file as the release body, so verifying it before merge is verifying the release notes. -- [ ] **The `cozystack/website` docs PR stays parked until publish.** Promotion opens it early; merging it before the GitHub release for `vX.Y.Z` is live 404s the new version's docs and API links. Merge it right after finalize publishes the release (regular-release step 6, patch step 7). - -## CI failures release engineers commonly hit - -| Symptom | Class | Diagnosis | Block release? | -|---------|-------|-----------|----------------| -| `Draft release for vX.Y.Z not found` on every release-* merge | Workflow regression — `Publish draft release` lost its `github-token` in a past refactor; default `GITHUB_TOKEN` cannot list drafts | At `pull-requests-release.yaml:163`. Undraft manually, file the fix | No | -| Changelog Copilot job returns 402 | Token quota — `COPILOT_GITHUB_TOKEN` premium-request quota empty | Both changelog entry points hit it: the optional rc-time `changelog-rc` dispatch and the promote-time `changelog` job (both call the reusable `changelog-generate.yaml`), plus the `tags.yaml` backstop | No — `continue-on-error: true` on the AI step, generate changelog by hand following [`agents/changelog.md`](./agents/changelog.md) | -| `Prepare Release` skipped and the run red, `Build arm64 images` failed | The rc's arm64 build failed | Read the `build-arm64` log; `build-arm64-nightly.yaml` usually failed the same way first. Fix on the line, cut the next rc | Yes — an rc is not published amd64 only | -| `Stitch multi-arch indexes and republish` fails listing refs | `verify-multiarch.sh` found refs that are not amd64+arm64 indexes, or that it could not check | Each listed ref names its reason. Single-platform manifest or index without one architecture: make the image multi-arch, or allowlist its repository with a reason if it cannot be. Manifest unreadable: correct the ref or digest; the allowlist does not excuse it. Rate-limited or unavailable after 3 tries: re-run once the throttle clears | Yes | -| E2E `kubernetes-test` fails with `CSINode does not contain driver csi.kubevirt.io` | Test ordering bug | Tenant's `kubernetes-${test_name}-csi` HelmRelease still installing when NFS PVC is created. Fix: wait for `csinode/` to advertise the driver before creating the PVC | Yes — flaky and masks real CSI bugs | -| `tenant-root` HR in perpetual upgrade-rollback loop after OIDC patch | Real bug at the intersection of #2602's readiness change and the cozystack reconcile cadence | Look for `rate: Wait(n=1) would exceed context deadline` in operator logs; multiple tenant child HRs stuck `InProgress` | Yes | -| `cozy-dashboard` ImagePullBackOff from `cozystack-ui:latest` | Registry flake | Known noisy on some OCI mirrors; not currently in the prepull set | No — retry | -| PR CI: `docker push` 409 on `:latest` for the same image across two PRs | Concurrent manifest race on a shared floating tag | The `:latest` tag was previously written by every `make image` regardless of PR. Resolved by the image-tag refactor (`IMAGE_TAG=pr--` in PR builds, `IMAGE_TAG=` only at release time) | No — `gh run rerun --failed` | - -**Heuristic:** workflow-token and Copilot-quota issues are out-of-band — release the tag, fix the workflow in a follow-up PR. Test ordering bugs and tenant-root reconcile storms are real and should block. - -If you find yourself doing the same manual fixup on two consecutive releases (e.g. "undraft the release"), open a workflow-regression issue. Workflow bugs with known manual workarounds rot silently for months. - -## Tag immutability - -Published tags are **write-once** — once a `vX.Y.Z` or rc tag is pushed it is never moved or deleted. (Nightlies are not git tags at all — they are GHCR OCI tags, see [Nightly builds](#nightly-builds).) Moving a tag silently poisons the Go module proxy / pkg.go.dev cache for `api/apps/v1alpha1/vX.Y.Z` and drifts SBOM/provenance toolchains, so the release flow is built so a move is impossible by construction: - -| File | Tag / branch handling | -|------|------------------------| -| [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) | Verifies the pinned packages candidate before creating a stable name. Creates `vX.Y.Z` at the PR merge commit **write-once** (create if absent, no-op if unchanged, fail loudly if it would move — the merge commit is new, so there is nothing to move). Cuts the `api/apps/v1alpha1/` Go-submodule tag write-once at the same commit, **stable only** (never rc/beta/alpha). The `release-X.Y` maintenance branch is fast-forward-only. Retags the verified packages artifact and rc container digests to the stable image tag (also write-once at the image level). | -| [`tags.yaml`](../.github/workflows/tags.yaml) | The `release-X.Y.Z-rc.N` staging branch is a mutable staging ref (compare-before-force: skipped when unchanged, force+log only when it genuinely moves). Fails fast on a stable `vX.Y.Z` tag with no pre-existing draft — stable tags come from `promote-rc.yaml`, never a hand push. | -| [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) | Mutates no stable tags. Before staging, verifies green full E2E from the latest rc tag-push run or a tag-correlated manual published-tag run (unless loudly bypassed). Commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning its digest on the `release-X.Y.Z` branch, generates the changelog there, and opens the promote PR with `release` but no automatic `full-e2e`; the PR gate verifies the candidate and finalize repeats that verification before the stable tag and image retag. The staging branch is force-pushed on re-dispatch, but an existing `docs/changelogs/vX.Y.Z.md` is preserved across the rebuild. | - -This is the immutable-tag + rc-promotion model from [#2677](https://github.com/cozystack/cozystack/issues/2677): stable `vX.Y.Z` is created only by promoting an existing rc, rc tags are write-once, and `api/apps/v1alpha1/vX.Y.Z` is created only on a stable release. The old nightly `auto-release.yaml` (which delete-recreated auto-bumped patch tags) has been removed. rc and stable tags accrete permanently; the only churning artifacts are GHCR nightlies and abandoned temporary promotion candidates, which [`retention.yaml`](../.github/workflows/retention.yaml) prunes (see [Nightly builds](#nightly-builds)) — GHCR storage is the cost vector to watch. - -### Repo-side enforcement (tag-protection rulesets) - -The guarantees above are enforced in the workflows. Repo admins back them with two GitHub **tag rulesets** targeting `v*`: - -- **Immutability** — *Restrict updates* + *Restrict deletions*, **no bypass**. No actor (not even the CI app) can move or delete a shipped `v*` tag. Safe to enable at any time: nothing in the release flow ever moves or deletes a `v*` tag, it only creates new ones. -- **Creation control** — *Restrict creations* with the **CI app as the sole bypass actor**, plus *Limit branches/tags updated in a single push* = 1. This turns "don't hand-push tags" from convention into hard enforcement: every `v*` tag — pre-release **and** stable — can then be created only by CI. Enable this **only after** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) is in use, since that workflow is what makes CI the sole creator of pre-release tags; enabling it earlier would lock maintainers out of cutting rc/alpha/beta. (The Go-module tag `api/apps/v1alpha1/*` gets an equivalent pair of rulesets.) - -## Splitting a release-blocking bundle PR - -Sometimes the work that has to land before a release is a 40-commit grab bag (CI stabilization, dependency bumps, chart fixes). Splitting it makes it reviewable and survivable. The strategy that has worked: - -1. **Verify what's already on `main`.** Use `git merge-base --is-ancestor` and subject grep to drop commits that have already landed. -2. **Find existing open PRs that cover the same commits.** Force-push to the existing branch instead of opening a duplicate — preserves review threads. -3. **Identify the "bottleneck" PR.** The one whose changes make previously-silent misconfigurations into hard errors (typically an operator-version bump). Everything else can fan out in parallel. -4. **Squash iterative same-author bug-on-bug fixes** inside each split. Preserve authorship via `--author` on the squashed commit. -5. **Mark the keystone PR as draft** and link in the body to the dependency PRs. Reviewers know it intentionally fails CI until the chain lands. -6. **Beware of concurrent registry pushes.** When fanning out 5+ PR CI runs simultaneously, multiple `make image` jobs can collide on shared floating tags. The `IMAGE_TAG=pr--` refactor solved this; if you see `Conflicted with another upload of the same manifest`, `gh run rerun --failed`. - -## Cleanup after release - -- The `release-X.Y.Z` branch is deleted by GitHub when its PR merges. The `release-X.Y` maintenance branch is created by `cut-prerelease.yaml` at the rc freeze (create-only, never force-moved) and fast-forwarded afterwards by `pull-requests-release.yaml::Ensure maintenance branch`. -- The draft release is published by the same workflow. -- The release body is set by finalize from `docs/changelogs/vX.Y.Z.md` at publication. Afterwards, `update-releasenotes.yaml` re-syncs that file into the GitHub Release body on any push to `main` that touches it — so edits to a published changelog land on the release page the next time `main` moves. -- Local cleanup: remove your worktree (`git worktree remove`) and prune merged release branches. +## Pages + +- [Introduction](./release/introduction.md): the three release types and the tag push that drives the pipeline. +- [Release Candidates](./release/release-candidates.md): what an rc is, and how cutting the first one freezes `release-X.Y`. +- [Regular Releases](./release/regular-releases.md): promoting an rc to `vX.Y.0`, step by step. +- [Patch Releases](./release/patch-releases.md): the same flow for `vX.Y.Z` on an existing `release-X.Y` line. +- [What CI does during the release process](./release/ci-pipeline.md): which workflow does what, phase by phase, including multi-arch builds and the rc E2E gate. +- [Stable tags come from rc promotion](./release/stable-tags.md): why a stable tag only ever comes from promoting an rc. +- [Release signatures and SBOM](./release/signatures-and-sbom.md): the source SBOM, checksums and cosign signature on each release, the SBOM each release image carries, and how to verify and read them. +- [Nightly builds](./release/nightly-builds.md): how a nightly mirrors `main` to GHCR, and how to install one. +- [Backports](./release/backports.md): the policy that decides what goes into a patch release, and how a contributor gets a fix there. +- [Backport mechanics](./release/backport-mechanics.md): cherry-pick triage before a patch, and the monotonic-counter trap. +- [Backport tasks](./release/backport-tasks/README.md): resolving a batch conflict, deferring the original a failed batch CI comes from, and checking a pull request title, with the check each has to pass. +- [Changelogs](./release/changelogs.md): changelog failure modes and the pre-publish checks. +- [Release-time fires](./release/release-time-fires.md): regressions that escaped to users, and the verification gaps they expose. +- [Pre-release verification checklist](./release/pre-release-checklist.md): what to verify before merging a release PR. +- [CI failures release engineers commonly hit](./release/ci-failures.md): known failures and whether each one blocks a release. +- [Tag immutability](./release/tag-immutability.md): write-once tags and the rulesets that enforce them. +- [Splitting a release-blocking bundle PR](./release/splitting-bundle-prs.md): how to break up a grab-bag PR that blocks a release. +- [Cleanup after release](./release/cleanup.md): what is left to clean up once a release is out. ## See also - [`agents/changelog.md`](./agents/changelog.md) — canonical changelog generation process. -- [`agents/contributing.md`](./agents/contributing.md) — commit/PR conventions, backport label semantics. +- [`agents/contributing.md`](./agents/contributing.md) — commit/PR conventions. - [`agents/releasing.md`](./agents/releasing.md) — pointer file for AI agents handling release tasks. - [`.github/workflows/cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — the sole entry point for cutting a pre-release tag as the CI app. - [`.github/workflows/tags.yaml`](../.github/workflows/tags.yaml) — tag-push pipeline (rc build + mandatory `rc-e2e`; stable-tag changelog/website backstops). @@ -714,5 +53,6 @@ Sometimes the work that has to land before a release is a 40-commit grab bag (CI - [`agents/image-refs.md`](./agents/image-refs.md) — where image references live, what their tags mean, and the invariants promotion and mirroring rely on. - [`hack/lib/image-refs.sh`](../hack/lib/image-refs.sh) — the single enumeration of ref-bearing files, shared by the promote, retag and mirror tooling. - [`.github/workflows/retention.yaml`](../.github/workflows/retention.yaml) — GHCR nightly and abandoned promotion-candidate pruning. -- [`.github/workflows/backport.yaml`](../.github/workflows/backport.yaml) — automatic cherry-pick bot. +- [`cmd/backport-batch`](../cmd/backport-batch/README.md) — the backport batch that applies the [backport policy](./release/backports.md). +- [`.github/workflows/backport-batch.yaml`](../.github/workflows/backport-batch.yaml) — runs the backport batch on Monday and Thursday for each line it runs for: the newest released lines, and a line in its rc window. - [`.github/workflows/update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — sync changelog → GitHub Release body. diff --git a/docs/release/backport-mechanics.md b/docs/release/backport-mechanics.md new file mode 100644 index 000000000..70da94e85 --- /dev/null +++ b/docs/release/backport-mechanics.md @@ -0,0 +1,37 @@ +## Backport mechanics + +Under the [backport policy](./backports.md) fixes reach a release line through the batch, and nobody labels pull requests for backport. This page covers the mechanics around it: cherry-pick triage before a patch, and the trap that keeps migrations out of patch releases. + +### Cherry-pick triage before a patch + +A patch release includes bugfixes for code that shipped in the corresponding minor `vX.Y.0`. What goes in is decided by the rules in [Backports](./backports.md), not per pull request, and each batch pull request says what it picked, what it deferred and what it skipped under which rule. What is left to check before a cut: + +```bash +# Exits 1 and prints the URLs while a pull request the rules take is not on +# the line, including one only on the open batch PR, which the cut would not +# carry; see cmd/backport-batch/README.md. +go build ./cmd/backport-batch +./backport-batch check --line release-X.Y --cut +``` + +[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check, also once that batch merged or closed. An original that waits on the open batch for a maintainer fails the cut. A change that reached the line by hand counts like any other, by what reached the line. + +### Backporting code that touches monotonic counters + +The batch never takes a change like this: rule 4 in [Backports](./backports.md) holds migrations, `targetVersion`, sizeable API changes and CRDs for the next minor. What follows is why, and what a maintainer has to do when a migration must reach a release line anyway. + +Migrations are the canonical case. `packages/core/platform/images/migrations/run-migrations.sh` is a linear counter — it runs `seq $CURRENT_VERSION $((TARGET_VERSION - 1))` and stops if `CURRENT >= TARGET`. This creates a forward-upgrade trap. Walking through it with `release-1.2` as the maintenance branch: + +- `main` has `targetVersion=10` and migrations 1–9 on disk. +- The `release-1.2` branch was cut when `targetVersion=5` (migrations 1–4 on disk). +- You backport a new migration to `release-1.2` at slot 5, and naively bump `targetVersion` 5 → 6 to enable it. +- Customer running 1.2.x runs the new migration, stamps `cozystack-version=6` in-cluster. +- On upgrade to 1.3 (main's `targetVersion=10`), the runner walks `seq 6 9` and runs main's migrations 6–9 — **main's own migration at slot 5 never runs.** The backport claimed that slot on `release-1.2`; main's slot 5 holds an unrelated migration that gets silently skipped. + +Mitigations, in order of preference: + +1. **Don't bump `targetVersion` in the backport.** Just ship the migration file. Prior precedent in the repo: an earlier maintenance branch's ACME backport did exactly this — the migration shipped but ran on no clusters until a later commit raised the counter. Pair this with relaxing the chart-side `{{ fail }}` guard on the maintenance branch (chart-side coalesce) so the new value works without the migration having executed. +2. **If you must bump,** add an idempotent duplicate of the now-skipped main migration at a higher slot. Bounded cost (one extra file per affected migration) and survives the skip window. +3. **Audit cross-branch counter state before deciding.** Compare migration numbering across `main`, `release-X.Y`, and `release-X.(Y-1)`. If a backport would create a skip window on any forward-upgrade path, choose option 1. + +This rule generalizes to any monotonic-counter state: schema versions, feature-flag generations, anything that gates "have I run X yet?" with a single integer. diff --git a/docs/release/backport-tasks/README.md b/docs/release/backport-tasks/README.md new file mode 100644 index 000000000..e4395480e --- /dev/null +++ b/docs/release/backport-tasks/README.md @@ -0,0 +1,29 @@ +## Backport tasks + +The [backport batch](../backports.md) is built by [`backport-batch`](../../../cmd/backport-batch/README.md), and two steps around it need a maintainer's judgement. Each is a task with fixed inputs, one allowed output and a check in code that the result passes before anything is pushed. Both wait on the batch pull request: the batch's comment lists the deferred conflicts, and a failed batch CI shows on its checks. + +| Task | Input | Output | Check | +|---|---|---|---| +| [Resolve a conflict](./resolve-conflict.md) | a deferred conflict on the batch pull request | the original's commits picked under one merge on the batch branch, or one pick for a squash | `backport-batch fold`, and `backport-batch verify` in CI | +| [Attribute a failed batch CI](./failed-ci.md) | the failing jobs of the batch pull request | the culprit deferred to the next batch | `backport-batch run --defer` | + +### Rules for every task + +- Push nothing but the branch of the open batch pull request, `backport-batch/-` in this repository, and only what the task's check accepted. Never push to `main` or a `release-*` branch, and never open a second batch pull request: the scheduled run continues the open one. +- Change no file the original pull request does not change. A conflict that cannot be resolved inside those files is not resolved: the original stays deferred. +- Never rewrite the batch branch by hand. A plain push to it is a fast-forward; only `backport-batch run --defer --push` replaces it, leased on the head it rebuilt. +- Leave the batch pull request's title and body to the tool: it rewrites both on every run, and a hand edit is lost or, inside the `backport-batch` block, makes `run`, `check`, `fold` and `notify` refuse the body, and inside the `backport-batch-report` block makes `run --push` and `backport-batch report` refuse it. Leave the tool's report comments as they are too: each keeps one classification of the line, and an edited one makes `backport-batch report` refuse it. +- When the task cannot be done within these rules, push nothing and leave a comment on the batch pull request saying why, so the next maintainer to look finds it. Nothing is lost by waiting: a red batch does not merge, and a deferred original goes to the next batch. + +### Getting the tool + +Build it from `main`, in a clean worktree of this repository whose `origin` is this repository and with `gh` authenticated as someone who can push to the batch branch: + +```bash +git fetch origin +git worktree add --detach ../backport-task origin/main +cd ../backport-task +go build -o /tmp/backport-batch ./cmd/backport-batch +``` + +Run it from that worktree. Every command but `notify` and `report` fetches `origin` first, and `run`, `check`, `fold` and `lines` read the policy from `origin/main`. diff --git a/docs/release/backport-tasks/failed-ci.md b/docs/release/backport-tasks/failed-ci.md new file mode 100644 index 000000000..18b26f4dc --- /dev/null +++ b/docs/release/backport-tasks/failed-ci.md @@ -0,0 +1,41 @@ +## Attribute a failed batch CI + +CI on the batch pull request runs the line's build and tests against every original the batch carries at once. When a check fails, this task finds the original the failure comes from and takes it out of the batch, so the rest can merge; the original goes to the next batch with the reason recorded. Read the [rules for every task](./README.md#rules-for-every-task) first. + +### Inputs + +- The open batch pull request for the line and its failing checks: `gh pr checks `, and the log of each failing job, `gh run view --log-failed`. +- The originals the batch carries, one unit each on its first-parent line: `git log --first-parent --reverse --format='%h %s' origin/..origin/backport-batch/-`. A merged pull request's unit is a merge whose message is the original's merge message, naming its number, and whose `Backported-from:` trailer names its merge commit on `main`; its commits are `git log --reverse --format='%h %an %s' ^1..^2`. A squash-merged one is one commit whose subject ends in its number. The files a unit changes are `git diff --name-only ^1 `. +- The line's own CI on its head, `gh run list --branch `, to tell a failure the batch brought from one the line already had. + +### Output + +The batch rebuilt without the culprits, by one `backport-batch run --defer` with `--push`, which records each culprit and the reason in the batch pull request. Nothing else: no commit by hand, no revert, no edit to the batch branch. + +### Steps + +1. Read the failing jobs and name what fails: the job, the package or test, and the first error. +2. When the same job fails the same way on the line's head, or the failure is the runner's (a timeout pulling an image, a lost runner, a registry error), the batch did not cause it: defer nothing, and comment on the batch pull request with the job and the line's run that shows the same failure. +3. Otherwise pin it on originals by their files: the originals that change the failing package, chart or test, or what it builds from. Take the fewest that explain the failure; when no original's files reach the failure, or every original's do, the culprit cannot be pinned from files alone: read the failure further before deferring anything. +4. Write the reason as one line naming the job and the failure, for example `e2e-apps: postgres never becomes Ready`. + +### Check before pushing + +```bash +/tmp/backport-batch run --line --defer N[,N] --reason ": " +``` + +It rebuilds the branch locally without those originals and prints the new body, exiting 0. It exits 2 and names the original when dropping one leaves a later commit that no longer applies; add that one to `--defer` only when its files reach the failure too, otherwise comment on the batch pull request naming it and stop. Then publish the same command with `--push`: + +```bash +/tmp/backport-batch run --line --defer N[,N] --reason ": " --push +``` + +It replaces the batch branch only if nobody pushed onto it since the rebuild read it, and updates the pull request body. CI runs again on the new head; if it fails again on another original, repeat from step 1. + +When the culprits are everything the batch carries, there is no branch left to push: the body printed without `--push` says the batch is closing, and `--push` records the deferrals in the batch pull request, closes it and deletes its branch, still only if nobody pushed onto it since. The next batch tries them again. If it stops part way, the next `run --push` on the line finishes the close, and says so. + +### Never + +- Defer an original the failure does not reach, to make the batch green by taking out more than the culprit. +- Push the rebuilt branch any other way, or push to it by hand. diff --git a/docs/release/backport-tasks/resolve-conflict.md b/docs/release/backport-tasks/resolve-conflict.md new file mode 100644 index 000000000..46e24cf8d --- /dev/null +++ b/docs/release/backport-tasks/resolve-conflict.md @@ -0,0 +1,66 @@ +## Resolve a conflict + +The batch carries a pull request merged into `main` as its own commits, each picked with `git cherry-pick -x`, under a merge commit that names the original; a squash-merged one as one pick. When any of an original's commits conflicts, nothing of it stays on the batch, and the batch pull request lists it under **Deferred conflicts** with the commit that failed and its files, and in its `backport-batch` block. This task ports one such original onto the batch by hand, in the same shape. Read the [rules for every task](./README.md#rules-for-every-task) first. + +### Inputs + +- The open batch pull request for the line, its branch `backport-batch/-` and its head commit, as GitHub shows them: `gh pr view --json headRefName,headRefOid`. The resolution lands on that batch, so without an open batch pull request there is nothing to resolve onto: wait for the next batch, which lists the conflict again, or port the change as a pull request of its own against the line, as [fixes that exist only on a release line](../backports.md#fixes-that-exist-only-on-a-release-line) describes, and drop the original from the line. +- The original's number `N` from the **Deferred conflicts** list, and its merge commit on `main`: `gh pr view N --json mergeCommit --jq .mergeCommit.oid`. It has two parents when the pull request was merged, one when it was squashed. +- For a merged pull request, its own commits, oldest first, without the merges that brought `main` into its branch: `git rev-list --reverse --topo-order --no-merges ^1..^2`. +- The files the original changes, `git diff --name-only ^1 `, which are the only files the resolution may touch. + +### Output + +For a merged pull request, one merge commit on the batch head: its first parent is the head of the batch pull request, and its second parent ends a straight line of picks of the pull request's own commits, in their order, each made with `git cherry-pick -x` so it keeps its author, message and sign-offs and ends in the `(cherry picked from commit …)` line git appends. Its message is the original's merge message with a `Backported-from: ` trailer of its own. For a squash-merged pull request, one pick of its commit. Either way only files the original changes, and nothing else: no extra commit, no fix-up of an earlier pick. + +### Steps + +```bash +git fetch origin +git switch --detach +base=$(git rev-parse HEAD) +``` + +Pick the original's commits, oldest first; for a squash-merged pull request the one commit to pick is its merge commit on `main`: + +```bash +git -c core.hooksPath=/dev/null cherry-pick -x --signoff +``` + +When a pick conflicts, resolve each conflicted file so that it does on the line what that commit does on `main`, keeping the line's own code wherever the commit does not change it, and go on with the next commit: + +```bash +git add +git -c core.hooksPath=/dev/null cherry-pick --continue --no-edit +``` + +When the original builds on something the line does not have, a helper, a value or another pull request, the resolution would have to bring that along, which this task does not allow: abort with `git cherry-pick --abort`, go back with `git switch --detach "$base"`, and comment on the batch pull request which change the original depends on; it stays deferred until that change is on the line. When that change will never reach the line, drop the original there instead, with `backport-batch run --line --drop N --reason "" --push`, so later batches stop trying it. A commit whose change the line already has stops the pick as empty; skip it with `git cherry-pick --skip`. + +For a merged pull request, join the picks to the batch head with the merge commit: + +```bash +tip=$(git rev-parse HEAD) +git switch --detach "$base" +{ git log -1 --format=%B ; printf '\nBackported-from: %s\n' ; } > /tmp/batch-merge-message +git -c core.hooksPath=/dev/null merge --no-ff -F /tmp/batch-merge-message "$tip" +``` + +### Check before pushing + +```bash +/tmp/backport-batch fold --line --original N --commit "$(git rev-parse HEAD)" +``` + +It prints `ok` and exits 0, or prints why not and exits 1. Push only on `ok`, as a fast-forward of the batch branch: + +```bash +git push origin HEAD:refs/heads/backport-batch/- +``` + +Resolve one original at a time and push each before starting the next: the next fold has to sit on the new head. CI on the batch pull request runs again on the push, `backport-batch verify` among it, and the next run of the batch lists the original as on the branch, with the picks the resolution changed marked as such. + +### Never + +- Push a commit `fold` did not print `ok` for, or push anywhere but the batch branch. +- Squash the original's commits into one pick, reorder them, or pick a commit of another pull request into the merge. +- Edit a file outside the original's files to make the pick build; that is the dependency case above. diff --git a/docs/release/backports.md b/docs/release/backports.md new file mode 100644 index 000000000..b0cefdd17 --- /dev/null +++ b/docs/release/backports.md @@ -0,0 +1,124 @@ +## Backports + +Backports are automatic. Everything that can go into a patch release goes in, and nobody decides it pull request by pull request: nobody requests a backport, and contributors never open backport pull requests. The one exception is a maintainer override, below. A batch job takes the fixes merged into `main` and cherry-picks them onto each `release-X.Y` line it runs for, following the rules on this page. + +### Getting a fix into a patch release + +Merge it into `main` with a `fix` title, for example `fix(postgres): keep the replica count on resize`. Nothing else is needed: if the change meets the rules below, the next batch picks it up for every [line the batch runs for](#which-lines-the-batch-runs-for). + +The batch takes a pull request's type from its title alone, so a fix that ships inside a `feat` or `refactor` pull request does not reach a patch release. Put the fix in a pull request of its own. + +### What the batch takes + +The batch takes a pull request from `main` into release line X.Y when all of these hold: + +1. It merged into `main` after `release-X.Y` branched, or after the line started under this policy if that is later (see [Where each line starts](#where-each-line-starts)), and the line does not carry it yet. +2. Its title type is `fix` or `security`, the type of a fix that comes through a security advisory. CI and e2e fixes count, because they keep the line's own CI working. The title is read once any leading `[Backport …]` prefixes are dropped, and a title that does not read as a Conventional Commits title is kept out. +3. It is not breaking: no `!` after the type or scope, and no `BREAKING CHANGE:` or `BREAKING-CHANGE:` footer at the start of any line of the description its merge into `main` recorded. A footer inside an HTML comment, where the pull request template quotes it, does not count. +4. It touches no platform migrations, anything under `packages/core/platform/images/migrations/migrations/`, and no `targetVersion` line of `packages/core/platform/values.yaml`, adds, removes or edits no CustomResourceDefinition document (a CRD in a test fixture under `tests/` or `testdata/` does not count, and neither does an edit to another document in the same file), and makes no change to the API that the [API review gate](../../.github/workflows/api-review-gate.yaml) calls sizeable: a new or removed API group or resource, or a schema change that can reject what the old schema accepted. Those wait for the next minor; a reworded description or a new optional field under `api/` does not. [Backporting code that touches monotonic counters](./backport-mechanics.md#backporting-code-that-touches-monotonic-counters) shows what a migration on a release line breaks. +5. Each of its commits picks cleanly with `git cherry-pick -x`, or its conflict is resolved during the batch, and the batch's build and tests pass. A pick that fails moves to the next batch, and the batch pull request lists it with the files that conflicted. +6. On an older line, it is already on the next newer line the batch runs for. Lines are processed newest first, so an older line takes only what the newer one already carries. + +The batch reads the pull requests off `main`'s first-parent history, one per merge commit or squash commit that names its pull request. A commit there that names none, a direct push for instance, and a merge of more than two parents are never taken; the batch pull request counts them. + +### What the batch leaves out on one line + +A pull request the rules take for every line can still be wrong for one of them, for instance because it builds on code that line does not have. A maintainer drops it from that line with `./backport-batch run --line --drop --reason --push`: the batch takes it off, if it carries it, and no later batch for that line picks it again. The drop is kept in the [line's record](#the-lines-record-and-reports), so it needs an open batch pull request to record it on; before a line's first batch pull request exists there is nothing to record it on, and the command refuses. `--undrop ` withdraws it, and the next batch tries the pull request again. A pull request the line already carries cannot be dropped. To land a hand-made port instead, open it as a pull request of its own, as [fixes that exist only on a release line](#fixes-that-exist-only-on-a-release-line) describes, and drop the original. + +### What waits for the next minor + +`feat`, `chore` and `docs` pull requests, and dependency bumps (scope `deps`), ship with the next minor, not in a patch release; only a maintainer's `backport/include`, below, takes one. If a class of fixes should also wait for a minor, it gets a new rule on this page; a single pull request that the rules get wrong is what the overrides below are for. + +### Maintainer overrides + +A maintainer overrides the rules for one pull request with a label on the merged pull request on `main`: + +- `backport/include` takes it although rules 2 to 4 keep it out, a `feat` for instance, or although it merged before its line started under this policy. It still has to pick cleanly or have its conflict resolved, pass CI on the batch, and, on an older line, already be on the newer one, and a line that already carries it does not take it again. +- `backport/skip` keeps it out although the rules take it, and `backport-batch check` stops asking for it. An open batch that picked it before the label came keeps it until `backport-batch run --defer` takes it out; the batch pull request says so meanwhile. + +A label decides for every line. To keep a pull request out of one line only, drop it there with `backport-batch run --drop`, as [what the batch leaves out on one line](#what-the-batch-leaves-out-on-one-line) describes; `backport/include` does not lift a drop. + +`backport/skip` wins when a pull request carries both. Each run reads the labels afresh, and the batch pull request lists every override and what became of it. + +### Which lines the batch runs for + +The batch runs for the two newest lines that have shipped a stable release, and for every line branched after the newest of them that has not shipped one yet. A line has shipped once its first stable `vX.Y.Z` tag exists, which only the merge of its promotion creates ([stable tags come from rc promotion](./stable-tags.md)); an rc, alpha or beta tag does not count. How many released lines it keeps is `released-lines` in the block below. + +With 1.6 and 1.7 released, cutting `v1.8.0-rc.1` creates `release-1.8` ([release candidates](./release-candidates.md)), and from the next run the batch runs for 1.6, 1.7 and 1.8. Once `v1.8.0` ships it runs for 1.7 and 1.8, and 1.6 takes no more batches. Rule 6 holds across all of them, newest first: while 1.8 is in its rc window, a fix reaches 1.7 only after the 1.8 batch that carries it has merged, and 1.6 only after 1.7's. + +During its rc window a line is frozen: every later rc is cut from `release-X.Y` itself, and `main` is open for the next minor. The freeze closes the line to `main`, not to the batch, which is how fixes reach it: a batch pull request against a frozen line is built, checked and merged like any other, and the next rc carries what it merged once `check --cut` passes (see [Running the batch](#running-the-batch)). A line that drops out keeps its open batch pull request as it is, and `backport-batch` no longer runs or checks that line; a maintainer merges the pull request or closes it. + +### Where each line starts + +The policy applies in full from `release-1.7` on: a line released before it took effect keeps the patch content its users chose it with, so it takes only fixes merged after the policy started, and an older one only with `backport/include`. The batch reads the block below from this page, on `main`: a line it lists starts on that day, and any other line starts where it branched from `main`. The block also gives the `batch-only` day and `released-lines`. + +From the `batch-only` day in the block, a change from `main` reaches a line only through the batch, and only the batch's picks show that a line carries it: a pick whose `(cherry picked from commit …)` line, the one git appends, names the original's merge commit or every commit of its own. A picked commit keeps its message and a batch merge carries the original's, so text in them, a `Backport of #N`, a `Backported-from:` trailer or an `-x` line the commit's author wrote, counts only on commits from before that day, where it records the hand backports and earlier batches. + +```backport-policy +# release-X.Y: YYYY-MM-DD, the day the line started under this policy +release-1.6: 2026-10-26 +# batch-only: YYYY-MM-DD, the day from which only the batch's picks show what a line carries +batch-only: 2026-10-26 +# released-lines: N, how many of the newest lines with a stable release the batch runs for +released-lines: 2 +``` + +### Fixes that exist only on a release line + +Some fixes go to a release line as a pull request of their own. Such pull requests are occasional, and maintainers open them: for a fix with no counterpart on `main`, for one that on `main` came as part of another pull request, or for one written for `main` rather than for the line. When the counterpart on `main` is a pull request of its own that the batch would take, the batch skips it as already on the line once picking it would change nothing there; otherwise a maintainer labels it `backport/skip`. + +A pull request opened against a `release-*` branch is closed automatically, with a comment that links to this page, when its author is a user account, not a bot, that GitHub names neither the repository's owner nor a member nor a collaborator, and whose access to the repository GitHub reports as read or none. Anyone with triage access or more keeps their pull request open. When GitHub's answer on the author's access cannot be read, the check closes nothing and fails, naming the read that failed. The check runs only when a pull request is opened, so a maintainer keeps a contributor's release-only fix by reopening it. [`hack/close-manual-backport.sh`](../../hack/close-manual-backport.sh) does this from [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml); run it with `--pr --dry-run` to see what it would decide for a given pull request when it is opened. + +### Cadence + +The batch runs for each of its lines twice a week, on Monday and Thursday: often enough that a fix reaches its line within days, rarely enough that nobody spends time on it every day, and early enough in the week that each run leaves working days to cut an rc, run its e2e and promote it before the weekend. Patch releases ship weekly. + +### Running the batch + +[`backport-batch`](../../cmd/backport-batch/README.md) builds the batch in a clean worktree, newest line first. The scheduled run below does this; by hand it is: + +```bash +go build ./cmd/backport-batch +./backport-batch run --line release-1.7 --continue --push +./backport-batch run --line release-1.6 --continue --push +``` + +A run by hand trusts the open batch pull request the scheduled run opened as `cozystack-ci[bot]` and continues it; `BACKPORT_BATCH_BOT` names another bot, and set empty trusts none. + +`./backport-batch lines` prints the lines, newest first. `run`, `check`, `fold` and `lines` read the policy block from this page on `main`, so until the page with its block is on `main` they need `--policy ` naming a ref that has it, and the scheduled run, which passes none, fails. + +`run` picks what the rules take onto `backport-batch/-` and opens a draft pull request against the line. A pull request merged into `main` with a merge commit goes onto the batch as its own commits, oldest first, each picked with `git cherry-pick -x --signoff` so it keeps its author, its message and its sign-offs and gains the sign-off of whoever runs the batch; commits that merged `main` into the pull request's branch are left out. A merge commit joins them to the batch: it carries the original's merge message from `main` and ends in a `Backported-from: ` trailer. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, nothing of that original stays on the batch. The batch pull request is merged into the line with a merge commit, so all of this reaches the line as it is. Its body lists the originals with how many commits each brought and which of them differ from the commit they came from, the conflicts it deferred with the commit that failed and its files, and what it skipped under which rule, and ends with the [line's record and the run's report](#the-lines-record-and-reports); `--push` prints the pull request's URL. While that pull request is open, the next run continues it, `--continue` reading its head, rather than opening another. CI on the batch pull request runs the line's build and tests, and a maintainer merges it. Since rule 6 reads the newer line itself, an older line takes a fix in the batch after the newer line's batch with that fix has merged. + +A deferred conflict waits for the next batch. To land it sooner, pick the original's commits onto the batch head with `git cherry-pick -x`, resolving each conflict, and join them to the batch head with a merge commit that names the original; an original squash-merged into `main` is one pick of its commit there, with no merge. Check that merge or that pick with `./backport-batch fold --line --original --commit `, and push it onto the batch branch, as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes. CI runs `backport-batch verify` on every push to a batch pull request, which holds every original on it to the same shape. When CI on the batch pull request fails, the original it comes from is taken out with `./backport-batch run --line --defer --reason --push` and goes to the next batch, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the rest of the batch merges without it. When it is all the batch carries, `--push` records why in the batch pull request, closes it and deletes its branch, and the next batch tries it again. `--drop` and `--undrop` change the record of the open batch pull request in the same way, and need one. + +Before a patch release or an rc is cut from a line, `./backport-batch check --line --cut` has to exit 0. A cut tags the line, so the check fails while a pull request the rules take is not on the line, including one that is only on the open batch pull request: merge the batch pull request first. It prints the URLs, and prints what the batch deferred as warnings, since deferring is a maintainer's call; so is a pull request labelled `backport/skip` that the open batch still carries. An original that waits on the batch for a maintainer fails the cut, also when it is all the batch has left. A cut that passes says so. Without `--cut`, as the scheduled run uses it, a change on the open batch pull request counts as carried. + +### The line's record and reports + +What the batch decided about a line, the conflicts it deferred, the originals taken out with `--defer` and those waiting for a maintainer, is kept in a `backport-batch` block in the body of the line's newest batch pull request, its record. `backport-batch` reads the record whether that pull request is open, merged or closed, and writes it only while the pull request is open: a merged or closed batch pull request is never edited or commented on again. So a conflict the batch deferred is still a warning in `check` after the batch merged, until a later batch tries it again. A run that picks nothing while no batch is open writes nothing, and says so. A record `backport-batch` cannot read, or one written by someone it does not trust, stops it rather than let it fall back to an older one; how it decides is in [its README](../../cmd/backport-batch/README.md#the-lines-record-and-reports). + +Each run also reports where every pull request merged into `main` since the line branched stands, and what it read to decide. The body carries the latest report under the record, and a comment the tool posts on the open batch pull request keeps each one, so an earlier classification is still there after a later run replaced the body, and after the batch merged. Each drop in a report keeps its reason and the merge commit it is bound to. `./backport-batch report --pr ` prints the report the body carries, and `--id ` an earlier one; it writes nothing and never fetches, and reads GitHub and, for a report that keeps drops, the clone's history at the heads the report was made from. A run that picks nothing while no batch is open has nowhere to keep its report: the next batch's report has the line's state. + +### Changes to workflows + +The scheduled run's token may not push a commit that changes `.github/workflows/`, so the scheduled run leaves such an original out (`run --no-workflow-files`). The batch pull request lists it under "Waiting for a maintainer", with the workflow files it changes, and so does the batch's comment. It is no conflict. The policy still takes it: `check` warns about it while it waits, and `check --cut` fails on it. When nothing else is picked there is no batch pull request to list it on, and `check` reports it as missing. + +A maintainer brings it over with their own token, one that may push workflow changes (the Workflows permission, or the classic `workflow` scope), by continuing the open batch without the flag: + +```bash +./backport-batch run --line --continue --push +``` + +That picks it onto the batch in the batch's shape, after what the batch carries, and the body stops listing it; later scheduled runs continue the batch on top. To cut without it instead, take it out with `./backport-batch run --line --defer --reason --push`. + +### The scheduled run + +[`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) on Monday and Thursday. For each line the batch runs for, newest first, it continues the open batch pull request or opens one, runs `check`, and then `backport-batch notify`, which reports what waits on the batch. A batch open for three days or more, so one the previous run already saw, gets a comment on its pull request saying so and a line in the run's summary. The comment is one per pull request, updated in place on every run. It also lists the deferred conflicts, which a maintainer resolves as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. A failed batch CI waits for a maintainer too, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the comment does not list it, and it shows on the batch pull request's checks. `hack/backport-batch-scheduled.sh --dry-run` does the same in the current worktree and changes nothing on GitHub. Its `check` sees the line without the batch the dry run just built, so it can exit 1 where the real run's check passes. + +The batch runs as the cozystack-ci GitHub App, which the release workflows already use, and needs no secret of its own. The job mints a token from `COZYSTACK_CI_APP_ID` and `COZYSTACK_CI_PRIVATE_KEY` for this repository alone, with Contents and Pull requests write and nothing more, and fails when either secret is missing. Its pushes start CI on the batch pull request, which a push with the job's own `GITHUB_TOKEN` would not. Its commits carry the App's bot, `cozystack-ci[bot]`, whose batch pull request and comment `backport-batch` trusts: `BACKPORT_BATCH_BOT`, which the job sets from the token, names it, and a run without the variable trusts `cozystack-ci[bot]` too. The App's installation has to grant this repository Contents and Pull requests read and write, which an organization admin sees among the organization's installed GitHub Apps. It has no Workflows permission, so a change to `.github/workflows/` waits for a maintainer, as [changes to workflows](#changes-to-workflows) describes. + +It also reads one repository setting, which a repository admin sets up: `BACKPORT_ALERT_TEAM`, a variable, the team or person the comment mentions, as `org/team` or a login. Unset, the comment mentions nobody: the batch pull request is a draft the App opened, so no one is notified of what waits unless they watch the repository. + +Optionally, once the `Backport batch shape` check runs from `main`, a ruleset makes it required on the release lines. It targets `release-*` and excludes `release-*.*.*`: `release-*` alone also matches the `release-X.Y.Z-rc.N` and `release-X.Y.Z` staging branches that [`tags.yaml`](../../.github/workflows/tags.yaml) and [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) push, which have no batch on them, and a required check would refuse those pushes. "Do not require status checks on creation" stays on, since [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` with a push at its first rc. + +To run the scheduled script for real by hand, export your own token as `BACKPORT_BATCH_TOKEN`: without `--dry-run` it uses that variable and no other credential. With a token that may push workflow changes, leave `--no-workflow-files` out and the run brings over what waits. diff --git a/docs/release/changelogs.md b/docs/release/changelogs.md new file mode 100644 index 000000000..f26ac56c5 --- /dev/null +++ b/docs/release/changelogs.md @@ -0,0 +1,107 @@ +## Changelogs + +### Where the canonical process lives + +[`agents/changelog.md`](../agents/changelog.md) is the source of truth. Read it end-to-end before generating; do not infer the process from past commits or memory. The CI runs it under Copilot via the reusable `changelog-generate.yaml` — called by `promote-rc.yaml::changelog` at promote time and by the optional `changelog-rc.yaml` at rc time — and, as a backstop, `tags.yaml::generate-changelog`; locally the same prompt is replayable by following the doc directly. + +### Common changelog failure modes + +These are mistakes that have shipped in real changelogs. Verify against each before merging the changelog PR. + +**1. Commits from outside the release range.** Caused by running `git log ..HEAD` from `main` while generating a patch changelog — `HEAD` on `main` contains everything that landed since the tag plus backports merged to `release-X.Y` after the tag was cut. Use `git log ..` explicitly. The doc was updated to require this after v1.3.1's changelog shipped with 14 entries for what was actually a 1-commit release. + +**2. Original + backport listed as two separate entries.** Each backport must coalesce with its original into a single entry of the form `(in #orig, backport #bp)`. To verify after generation: + +```bash +# Every entry that mentions "backport" should also reference the original PR number +grep -E 'backport #[0-9]+' docs/changelogs/v.md | grep -v '#[0-9]*, backport' +``` + +Output should be empty. + +**3. Wrong PR author.** The squash-merge commit's author is whoever clicked "Merge" — not the person who wrote the code. **Always** resolve via `gh pr view --json author --jq .author.login`, never `git log --format=%an`. This bites hardest for website-repo entries where the same merger handles most PRs. + +**4. Superseding patch, stale changelog.** Tags are write-once, so a critical fix after `vX.Y.Z` ships as the next patch `vX.Y.(Z+1)` rather than moving the tag. If a changelog PR already exists for a tag you are superseding, the changelog work transfers to the new patch. Compare `git log ..` against what the existing `changelog-vX.Y.Z` branch already documents; only add/remove the deltas. If the only new commits are CI-internal or a revert of a feature that never reached a stable tag, no changelog edit is needed. + +**5. PR numbers swapped inside prose.** The entry-format validator checks bullet entries but ignores Feature Highlights paragraphs and Upgrade Notes. Both have caused wrong PR references in shipped changelogs. Verify every `#NNNN` in prose with `gh pr view `. + +**6. Hallucinated entries.** Every entry must correspond to a commit in `git log ..`. Verify by extracting PR numbers from the file and grep-searching the commit range: + +```bash +grep -oE '#[0-9]+' docs/changelogs/v.md | sort -u | while read pr; do + n=${pr#\#} + if ! git log .. --grep="#${n}" --oneline | grep -q .; then + echo "PR $pr in changelog but not in commit range" + fi +done +``` + +**7. Bot accounts as human contributors.** `cozystack-ci[bot]`, `github-actions`, `dependabot`, `renovate`, and any `app/*` login must NOT appear in the `## Contributors` list. They legitimately appear in **per-entry attribution** (e.g. a Renovate-authored PR) — but not in the human roll-call. + +**8. Title duplicated as description.** `* **fix(foo): X**: fix(foo): X (...)` means you wrote the conventional-commit subject twice and never wrote a user-facing description. The brief and the detail must say different things — the detail explains what the change means for users. + +### Changelog pre-publish verification + +Run these checks before merging the changelog PR. Catches problems in roughly the order they occur: + +```bash +PREV=v1.2.0; NEW=v1.2.1 + +# A. Every chart bump is mentioned somewhere in the changelog +git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*/*/charts/*/Chart.yaml' | while read f; do + old=$(git show "$PREV:$f" 2>/dev/null | yq '.version // ""') + new=$(git show "$NEW:$f" 2>/dev/null | yq '.version // ""') + if [ "$old" != "$new" ]; then + component=$(basename "$(dirname "$f")") + if ! grep -qi "$component" docs/changelogs/$NEW.md; then + echo "MISSING: $component $old -> $new ($f)" + fi + fi +done + +# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images) +for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do + git diff $PREV..$NEW -- "$path" | head -40 +done +git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile' + +# C. Every cited #NNNN exists and is a merged PR (not an issue) +grep -oE '#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's/#//' | xargs -P 20 -I{} sh -c ' + state=$(gh api /repos/cozystack/cozystack/issues/{} --jq ".pull_request.merged_at // \"ISSUE\"" 2>/dev/null) + echo "#{} $state" +' | grep -E 'ISSUE|null|^#[0-9]+ $' + +# D. URL liveness +grep -oE 'https://[^)" ]+' docs/changelogs/$NEW.md | sort -u | xargs -P 10 -I{} sh -c ' + code=$(curl -s -o /dev/null -w "%{http_code}" {}) + if [ "$code" != "200" ]; then echo "$code {}"; fi +' + +# E. Website PR authors are PR authors, not commit authors +grep -oE 'cozystack/website#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's|cozystack/website#||' | while read n; do + pr_author=$(gh pr view $n --repo cozystack/website --json author --jq .author.login) + echo "website#$n -> @$pr_author" +done + +# F. Working tree clean except for the changelog +git status --porcelain | grep -v "docs/changelogs/$NEW.md" +# Should output nothing. +``` + +Ship criterion: every chart bump surfaced, no issue-cited-as-PR, no 404 URLs, every website entry attributed via `gh pr view` (not commit author), `git status` shows only the changelog. + +### Cozystack-specific URL quirks + +`cozystack.io` paths are versioned: `/docs/operations/foo/` and `/docs/next/foo/` both 404. Only `/docs/vX.Y/foo/` and the unversioned canonical `/docs/foo/` serve content. AI-generated changelogs invent the unversioned form. Verify URLs with the `curl` loop above. + +### Platform-component coverage + +For a minor release (`vX.Y.0`), changelog entries should surface what an upstream chart bump actually brought in — features, breaking changes, security fixes. PR commit messages usually don't capture this; the upstream `CHANGELOG.md` or GitHub Release notes do. For each bumped `Chart.yaml`: + +1. Read the `sources:` field — typically a GitHub URL. +2. Fetch release notes strictly inside `old < x <= new` via `gh release view --repo / --json body,name`. +3. Summarize in 2–5 user-impact bullets per component (new features, breaking changes, CVEs, important deprecations). Don't reproduce upstream notes verbatim; link to them. +4. If a PR-level entry already mentions the bump, enrich that entry with bullets — don't duplicate the entry. +5. If 3+ bumps have no PR-level mention, introduce a `## Platform Components` section. + +If you can't find upstream notes for a bumped component, list it with version-only info and no bullets. Hallucinated bullets are worse than missing detail. diff --git a/docs/release/ci-failures.md b/docs/release/ci-failures.md new file mode 100644 index 000000000..9beabe02c --- /dev/null +++ b/docs/release/ci-failures.md @@ -0,0 +1,16 @@ +## CI failures release engineers commonly hit + +| Symptom | Class | Diagnosis | Block release? | +|---------|-------|-----------|----------------| +| `Draft release for vX.Y.Z not found` on every release-* merge | Workflow regression — `Publish draft release` lost its `github-token` in a past refactor; default `GITHUB_TOKEN` cannot list drafts | At `pull-requests-release.yaml:163`. Undraft manually, file the fix | No | +| Changelog Copilot job returns 402 | Token quota — `COPILOT_GITHUB_TOKEN` premium-request quota empty | Both changelog entry points hit it: the optional rc-time `changelog-rc` dispatch and the promote-time `changelog` job (both call the reusable `changelog-generate.yaml`), plus the `tags.yaml` backstop | No — `continue-on-error: true` on the AI step, generate changelog by hand following [`agents/changelog.md`](../agents/changelog.md) | +| `Prepare Release` skipped and the run red, `Build arm64 images` failed | The rc's arm64 build failed | Read the `build-arm64` log; `build-arm64-nightly.yaml` usually failed the same way first. Fix on the line, cut the next rc | Yes — an rc is not published amd64 only | +| `Stitch multi-arch indexes and republish` fails listing refs | `verify-multiarch.sh` found refs that are not amd64+arm64 indexes, or that it could not check | Each listed ref names its reason. Single-platform manifest or index without one architecture: make the image multi-arch, or allowlist its repository with a reason if it cannot be. Manifest unreadable: correct the ref or digest; the allowlist does not excuse it. Rate-limited or unavailable after 3 tries: re-run once the throttle clears | Yes | +| E2E `kubernetes-test` fails with `CSINode does not contain driver csi.kubevirt.io` | Test ordering bug | Tenant's `kubernetes-${test_name}-csi` HelmRelease still installing when NFS PVC is created. Fix: wait for `csinode/` to advertise the driver before creating the PVC | Yes — flaky and masks real CSI bugs | +| `tenant-root` HR in perpetual upgrade-rollback loop after OIDC patch | Real bug at the intersection of #2602's readiness change and the cozystack reconcile cadence | Look for `rate: Wait(n=1) would exceed context deadline` in operator logs; multiple tenant child HRs stuck `InProgress` | Yes | +| `cozy-dashboard` ImagePullBackOff from `cozystack-ui:latest` | Registry flake | Known noisy on some OCI mirrors; not currently in the prepull set | No — retry | +| PR CI: `docker push` 409 on `:latest` for the same image across two PRs | Concurrent manifest race on a shared floating tag | The `:latest` tag was previously written by every `make image` regardless of PR. Resolved by the image-tag refactor (`IMAGE_TAG=pr--` in PR builds, `IMAGE_TAG=` only at release time) | No — `gh run rerun --failed` | + +**Heuristic:** workflow-token and Copilot-quota issues are out-of-band — release the tag, fix the workflow in a follow-up PR. Test ordering bugs and tenant-root reconcile storms are real and should block. + +If you find yourself doing the same manual fixup on two consecutive releases (e.g. "undraft the release"), open a workflow-regression issue. Workflow bugs with known manual workarounds rot silently for months. diff --git a/docs/release/ci-pipeline.md b/docs/release/ci-pipeline.md new file mode 100644 index 000000000..73f105bb7 --- /dev/null +++ b/docs/release/ci-pipeline.md @@ -0,0 +1,107 @@ +## What CI does during the release process + +The numbered process of a [regular release](./regular-releases.md#regular-releases) is implemented by seven workflows, including the reusable/manual [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml), plus the reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml) that two of them share to produce the changelog. Knowing which job does what makes the failure modes much easier to diagnose. + +1. [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) — manual dispatch (from `main` or `release-X.Y`): the sole entry point for creating a pre-release tag. Validates the `-alpha`/`-beta`/`-rc` tag, then pushes it (write-once, at the branch tip) as the CI app so `tags.yaml` fires. Cutting the first `vX.Y.0-rc.N` also creates `release-X.Y`, freezing the line; afterwards a `vX.Y.*` dispatch from `main` is refused. Stable tags are never created here. +2. [`tags.yaml`](../../.github/workflows/tags.yaml) — fires on any pre-release tag push: `build-amd64` and `build-arm64` build the pre-release side by side, then `prepare-release` publishes it and pushes its digest-pinned staging branch, then — **only for `-rc.N` tags** — the mandatory `rc-e2e` job calls `e2e-tag.yaml` in the same run (`-alpha.N` / `-beta.N` skip it and are validated on demand with `e2e-tag.yaml`'s manual button). `generate-changelog` and `update-website-docs` also live here but are **backstops only** — both are normally produced at promotion time (see 4); on a stable tag push each self-skips or no-ops when the promote flow already did the work. These stable-tag backstops only fire because finalize pushes the stable tag with the CI app token (a `GITHUB_TOKEN` push triggers no run — see 5). +3. [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) — reusable from `tags.yaml::rc-e2e` and manually dispatchable for a published tag. It checks that the tag's release carries its nocloud disk, resolves the digest-pinned tree and runs the full install, OpenAPI, and Chainsaw suite on Talos containers. A tag cut before the container lane existed has to be dispatched from its own ref. Its dispatch run title is the constant workflow name, so the promote gate correlates a manual run through the successful tag-bearing E2E job name rather than the title. +4. [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) — manual dispatch, from `main` only (the signature over a stable release's assets names that ref, see [Release signatures and SBOM](./signatures-and-sbom.md#release-signatures-and-sbom)). Five jobs: `parse` validates the rc tag and requires green full-E2E evidence before any staging; `promote` rewrites and commits the rc tags, publishes that packages tree as a run-specific temporary reproducible candidate, commits its digest pin in the `release-X.Y.Z` tree, and drafts the stable release; `changelog` calls the reusable `changelog-generate.yaml` **in parallel** to produce `docs/changelogs/vX.Y.Z.md` (regenerating it, or copying an rc-time changelog already committed to the `release-X.Y.Z-rc.N` staging branch by `changelog-rc.yaml` — see 7); `website-docs` (needs `parse`, `promote`) opens the `cozystack/website` docs PR from the staging branch via `FETCH_REF` (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); `open-pr` joins them, commits the changelog onto the staging branch, and opens the `release-X.Y.Z` promote PR with the `release` label but no automatic `full-e2e`. So the changelog and website docs are reviewed as part of the promotion, the already-checked rc suite is not repeated by default, and the PR body carries ✅ verified or ⚠️ bypassed plus ✅/⚠️ changelog and website-docs lines. Both `changelog` and `website-docs` are non-blocking. No stable registry name is created at dispatch. +5. [`.github/workflows/pull-requests.yaml`](../../.github/workflows/pull-requests.yaml) verifies a promotion candidate against the prospective merge tree and original rc artifact before the required `E2E Tests` status can turn green. [`pull-requests-release.yaml`](../../.github/workflows/pull-requests-release.yaml) fires after merge and repeats that verification before any stable tag, then cuts the write-once stable + Go-module tags, publishes the release **with the merged changelog as its body**, retags the candidate and rc images to stable by digest (`:latest` gated on newest-stable), and publishes the stable chart. +6. [`update-releasenotes.yaml`](../../.github/workflows/update-releasenotes.yaml) — fires on pushes to `main` that touch `docs/changelogs/v*.md`; syncs that content into the corresponding GitHub Release body. No longer the primary path (finalize sets the body directly, which is race-free and works for maintenance lines too) — this now covers later edits to a published changelog and manual re-syncs via `workflow_dispatch`. +7. [`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml) — manual dispatch, **optional**: generate the changelog at rc time instead of waiting for promotion, while there is still time to review it. It calls the same reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml) and commits the result to the `release-X.Y.Z-rc.N` staging branch as `docs/changelogs/vX.Y.Z.md`; the promote-time `changelog` job then copies that file instead of running the AI again. Skipping this workflow changes nothing — promote generates the changelog itself, exactly as before. + +### Phase 1 — `build-amd64`, `build-arm64` and `prepare-release` (hard gate) + +On an rc tag push: builds images for both architectures, commits digest pins, creates the draft release, and pushes the `release-X.Y.Z-rc.N` staging branch. `build-amd64` holds the release-exists no-op and the rejection of a hand-pushed stable tag, and builds the amd64 images and the release assets (its `make build` also pushes the packages artifact and the installer chart, pinned on amd64 digests); `prepare-release` stitches, gates, republishes those two from the repinned tree, and publishes the release. (The stable promote PR is opened later by `promote-rc.yaml`, not here. When the stable tag itself is pushed by finalize, a draft already exists, so this job is a no-op — see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize).) + +The commit (`Prepare release vX.Y.Z`, authored by `cozystack-ci[bot]`) is **digest pins and image tags only**: + +- `packages/core/{installer,platform,testing}/values.yaml` +- `packages/system/*/values.yaml` (cozystack-api, dashboard, kamaji, linstor, kubevirt-csi-node, etc.) +- `packages/apps/kubernetes/images/{kubevirt-csi-driver,ubuntu-container-disk-*}.tag` +- `packages/system/dashboard/templates/configmap.yaml` (the `$tenantText` value) + +Things that look surprising in the diff but are normal: + +- **Zero-diff packages**: when buildx fully caches and upstream base images haven't moved, the digest is byte-identical to the previous release and won't appear. Verify the build step actually ran — don't assume "no diff = nothing built." +- **Sudden churn on `ubuntu-container-disk-*` tags**: `cloud-images.ubuntu.com/noble/current/` is a moving target; these often rebuild even without code changes. +- **Switched registry**: if a self-built image moved from `docker.io` to `ghcr.io`, the registry portion of the digest changes — that's a deliberate move, not a regression. +- **`targetVersion` is NOT touched by the release PR.** Platform migration `targetVersion` is bumped earlier in a feature commit (e.g. `[platform] Bump migration targetVersion to 39 for migration 38`). The release PR only re-pins the `platform-migrations` image digest. + +#### Multi-arch images + +Every pre-release is published for amd64 and arm64. `tags.yaml::build-arm64` builds the images on a native arm64 runner and pushes each as `-arm64`, while `build-amd64` builds amd64, the Talos installer and matchbox for both arches, and the assets on x86 at the same time. `build-amd64` hands its stamped tree, the log of the tags it pushed and the assets to `prepare-release` as workflow artifacts; a missing one fails the upload or the download rather than publishing a partial release. `prepare-release` waits for both builds and runs only if both succeeded: when either fails, the rc is not prepared at all, so a release is never published amd64 only. It applies the stamped tree and runs [`hack/stitch-multiarch.sh`](../../hack/stitch-multiarch.sh), which joins each image with its arm64 twin into one index, moves onto it every tag the build pushed for that image (the rc tag and, under `PUBLISH_VERSIONED=1`, its component version, which the build records in `PUSHED_TAGS_LOG`), and repins the tree on the index digests, then [`hack/verify-multiarch.sh`](../../hack/verify-multiarch.sh), then republishes the packages artifact and the installer chart from the repinned tree. A stable tag skips `build-arm64` and builds nothing: promotion copies the rc's indexes with every platform, so the stable release is exactly as multi-arch as its rc. Because the two builds run in parallel, an rc that is not prepared (its arm64 build failed, or `prepare-release` failed before the stitch) still leaves behind what the amd64 build pushed: amd64-only images under `:` and under the component-version tags, and the `cozystack-packages:` artifact and the installer chart, both pinned on those amd64 digests. A component tag an earlier rc had moved to an index then points at an amd64-only image again, until a run that succeeds stitches it, a re-run of the same run included. + +`verify-multiarch.sh` is a hard gate. It resolves every ref the tree pins, third-party images included, and fails naming each one that is not an index carrying both `linux/amd64` and `linux/arm64`, or that it cannot resolve at all. The fix is to make the image multi-arch (build it for both, or pin a multi-arch digest of the same version); an image that cannot exist for arm64 goes in [`hack/multiarch-allowlist`](../../hack/multiarch-allowlist) by repository, with the reason on the same line. + +That gate sees only what the tree pins by digest. An image a vendored chart renders from its own defaults, or one an operator launches at runtime, is invisible to it. `rc-e2e` closes most of that gap: before tearing the sandbox down, [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) reads every image the three Talos nodes pulled from their image stores (`hack/collect-images.sh`), each by the digest the pull resolved to, and runs `verify-multiarch.sh --refs-file` on that list with the same allowlist. A pre-release fails on any single-arch image, on any ref it cannot read or that is still rate-limited after the retries, and on a list the collection could not produce; a stable tag's run only warns, since it ships its rc's images. A tag whose tree predates the audit (it has no `hack/verify-multiarch.sh`) only warns and passes, so an rc cut before the audit existed can still be re-validated. The list and the audit output are in the run's `image-list-` artifact. Neither check covers a package that e2e never installs and whose images the tree does not pin by digest: an arm64 break there shows up only on an arm64 cluster. + +[`build-arm64-nightly.yaml`](../../.github/workflows/build-arm64-nightly.yaml) builds `main` for arm64 every day, so an arm64 build break shows up there before the next rc. Its first step prints `verify-multiarch.sh --report` for `main`'s committed tree: `main` pins amd64 images, so its first-party refs are all listed and are expected to be, since the rc's stitch turns them into indexes. A third-party ref that an in-tree rebuild stamps over at build time, as the Harbor components and shadowbox are, is listed too and goes away on the rc. The lines that matter are the third-party refs no build replaces and refs that cannot be resolved; each of those fails the next rc until it is fixed or allowlisted. matchbox is listed too and does not matter: `build-amd64` rebuilds it as a two-arch index. The stitch names the images it skips in its own summary: an image with no `-arm64` twin, one whose digest is also pinned in a file the rewrite does not reach, and a pair that is not one amd64 and one arm64 image. matchbox (already an index) and the e2e sandbox (never built for arm64, and allowlisted) are skipped on every rc and pass the gate; any other skipped image fails it. + +### RC E2E — mandatory check before promotion + +For an rc, `prepare-release` must finish before `tags.yaml::rc-e2e` calls the published-tag lane, so the full suite tests the digest-pinned staging tree that promotion will reuse, and the release it runs against already carries its disk. `promote-rc.yaml::parse` then checks the latest `tags.yaml` push run whose `head_branch` is the rc tag and accepts it only when the reusable E2E job succeeded. GitHub exposes called-workflow job names as `E2E Release Candidate / E2E vX.Y.Z-rc.N (full suite)`, which the gate matches by the exact tag-bearing suffix. + +The alternate evidence path scans retained successful `workflow_dispatch` runs of `e2e-tag.yaml` and accepts one whose direct E2E job is named `E2E vX.Y.Z-rc.N (full suite)`. The workflow's run title is always `E2E Release Tag` and therefore cannot correlate the input tag; artifact names also carry the tag but expire independently, so the job name is the strongest available correlator. If Actions retention or manual deletion removes the run/job metadata, or that job is renamed without updating the matcher, the gate fails closed and tells the maintainer to run the E2E Release Tag button again. `skip_e2e_gate=true` is an emergency override, not alternate evidence: it writes a loud warning and a ⚠️ PR-body line. + +### Phase 2 — the changelog (generated at rc time or promotion, not here) + +The changelog for `vX.Y.Z` is produced by [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml)'s `changelog` job (a thin call to the reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml)) and committed onto the `release-X.Y.Z` staging branch by `open-pr`, so it rides the promote PR and is on the base branch the moment the release publishes. Finalize then uses that file verbatim as the GitHub Release body. Generating it here — on the stable tag push — would be too late by construction: that tag is created by finalize, which has already published the release. It can also be produced earlier, at rc time, via the optional [`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml) button — which runs the same reusable core and commits the file to the `release-X.Y.Z-rc.N` staging branch, from which the promote-time `changelog` job copies it (validated) instead of regenerating. + +The `changelog` job runs **in parallel** with `promote`, detached at the rc tag (the "promotion" configuration in §2 of [`agents/changelog.md`](../agents/changelog.md)), using a separate **read-only** GitHub App token for the AI step so the model cannot mutate the repo even with `--allow-all-tools`. Every write — the commit onto the staging branch and the PR — happens in `open-pr` under a write-scoped token. + +Generation is deliberately **non-blocking**: a Copilot outage or exhausted token quota must never block a release. Only the AI step carries `continue-on-error`, and `open-pr` is gated on `promote` alone (`if: !cancelled() && needs.promote.result == 'success'`) so the promote PR opens regardless. When the changelog is missing the PR body says so explicitly — that ⚠️ line is the signal to act on. + +Recovery, in order of preference: + +1. Commit `docs/changelogs/vX.Y.Z.md` to the `release-X.Y.Z` branch by hand (follow [`agents/changelog.md`](../agents/changelog.md)). `promote` preserves it across a re-dispatch, and `open-pr` will not overwrite it with generated output. +2. Re-dispatch `promote-rc.yaml` — generation runs again; anything already on the branch wins. +3. Merge without one and let `tags.yaml::generate-changelog` (the backstop) open a changelog PR against `main` after the fact. The release is published with the draft's body until that merges. + +Known failure modes: + +- Copilot quota exhausted (`COPILOT_GITHUB_TOKEN` 402). Refill or rotate, then re-dispatch. +- AI step timeout (hard 30-min cap). +- Output truncated or whitespace-only — caught by the `Verify changelog` step (`hack/validate-changelog.sh`), which accepts either header convention (`# Cozystack vX.Y.Z` for a minor or `# vX.Y.Z ()` for a patch), and requires the leading release-link comment pointing at `releases/tag/vX.Y.Z`, a `compare/...vX.Y.Z` link ending in this version, and at least one `## ` section — deliberately with no line-count floor, since a complete short patch changelog (v1.5.1 ships in 19 lines) must not be rejected as a fragment. It downgrades to the missing-changelog path rather than shipping a fragment as release notes. + +**`tags.yaml::generate-changelog` is the backstop.** It self-skips when the changelog is already on `main`. It earns its keep in two cases: generation failed during promotion, or the promote PR targeted `release-X.Y` so the file never reached `main`. Since the rc freeze the second case is **every** release, minor and patch alike — the promote PR always targets the release line (see [Regular Releases](./regular-releases.md#regular-releases)), which makes this backstop the only route by which a published changelog reaches `main`. In that case it **ports the reviewed file from the tag commit verbatim** rather than regenerating — a second AI pass there would spend quota and then overwrite already-published, already-reviewed release notes. + +### Phase 3 — `update-website-docs` (now the backstop) + +The `cozystack/website` "update managed apps reference" PR is normally opened **earlier, at promote time**, by [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml)'s `website-docs` job — not here. That job fetches the README content from the `release-X.Y.Z` staging branch `promote` just pushed (via the website Makefile's `FETCH_REF` variable), so the docs are ready to review alongside the promote PR while the stable tag is still unborn. It refreshes the trunk version pins from the staging ref before promoting `next/` → `vX.Y/`, so a new minor's docs inherit fresh Talos/cozystack pins instead of a stale `next.yaml`. It is **non-blocking**: if it fails (or trips its `FETCH_REF`-support guard against a website checkout that predates that Makefile variable), the promote PR opens with a ⚠️ website-docs line and this tag-time job backfills the same branch. + +**Merge-timing contract:** the website PR must NOT be merged before the release is published (finalize). Merging it early flips the site's latest-version pointer to a version whose versioned docs and `openapi.json` are not on GitHub yet, so every version-pin and API link for the new version 404s. Both the website PR body and the promote PR body say so; park the website PR until the GitHub release for `vX.Y.Z` is live, then merge. + +This tag-time `update-website-docs` job stays as the **backstop**. It runs with `if: !cancelled() && needs.prepare-release.result == 'success' && !contains(github.ref_name, '-')` (surviving a failed/skipped changelog phase, stopping on cancel, and skipping every pre-release tag), decides whether to promote `next/` → `vX.Y/`, runs `make update-all`, stages `content hugo.yaml data/versions`, and force-pushes the same `update-docs-vX.Y.Z` branch. On the normal path — where `promote-rc.yaml::website-docs` already did the work — it no-ops into "No changes to commit". At tag time the stable tag exists, so its `BRANCH=RELEASE_TAG` fetch resolves without needing `FETCH_REF`. For it to fire at all, finalize must push the stable tag with the **CI app token** (see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize)) — a tag pushed with the default `GITHUB_TOKEN` triggers no workflow run. + +If anyone changes the website Makefile to write somewhere else (e.g. `static/`, `i18n/`), the `git add` list must grow in **both** jobs or the PR silently drops files. + +### Phase 4 — review and merge the promote PR + +Reviewer checklist for the `chore(release): promote vX.Y.Z-rc.N -> vX.Y.Z` PR: + +- [ ] The PR body says `✅ RC full e2e was verified`; if it says the gate was bypassed, obtain explicit maintainer acceptance or add `full-e2e` and wait for that opt-in run. +- [ ] `Verify release packages candidate` is green. It pulls the candidate and original rc artifact by digest, compares the candidate to GitHub's prospective merge tree, and feeds the required `E2E Tests` status even when full E2E is not repeated. Do not edit `packages/` on this PR after generation; if the guard reports drift, re-dispatch promotion to create and pin a new run-specific candidate. Note what a green here does *not* cover: GitHub fires no `pull_request` event when the base branch advances, so a `packages/` change merged to the base afterwards is never re-verified on the PR. Finalize repeats the check against the real merge commit and catches that case before any stable name exists — but it catches it after the promote PR has already merged, so a `packages/` change landing on the base while a promote PR is open is worth holding back. +- [ ] The PR body's **website-docs line** is present: if ✅, the `cozystack/website` docs PR is open and **parked** — do NOT merge it until `vX.Y.Z` is published (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); if ⚠️, the tag-time backstop will backfill it after finalize. +- [ ] Diff is digest pins + image tags only — nothing else. +- [ ] No accidental config drift (a value file you don't recognize). +- [ ] If any chart bumped, check the new digest pulls — `crane manifest ghcr.io/cozystack/cozystack/@` should return. +- [ ] Build artifacts on the draft release page exist and match the expected count. +- [ ] The PR includes `docs/changelogs/vX.Y.Z.md`, it has content, and the entries match the commit range (see [Changelog pre-publish verification](./changelogs.md#changelog-pre-publish-verification)). If the PR body carries the ⚠️ "No changelog in this PR" line, resolve that before merging — finalize publishes this file as the release body, unless the PR also carries `docs/releasenotes/vX.Y.Z.md` (see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). + +### Phase 5 — `pull-requests-release.yaml` (finalize) + +Fires on merge of a PR that is merged, carries the `release` label, and is authored by `cozystack-ci[bot]` (the last guard closes the "name a branch `release-X.Y.Z`, get it labelled, merge it" hole). Head branch must match `release-X.Y.Z[-suffix]`. This is where **every consumer-visible or write-once side effect of a promotion happens** — after the rc's mandatory E2E was checked at dispatch and the PR merged, making promotion transactional at stable names. Steps: + +1. **Verify the packages candidate before creating any stable tag.** [`hack/verify-promoted-packages.sh`](../../hack/verify-promoted-packages.sh) pulls the candidate by the digest committed in installer values, proves it matches the merged `packages/` tree after normalizing only the impossible self-reference, rejects surviving rc image tags, then follows the candidate's embedded pre-push `platformSourceRef` back to the original rc artifact and compares normalized container `repo@digest` sets. Any package edit after candidate publication or any changed container digest aborts finalize while all stable names are still absent. +2. **Create the tag at the merge commit** (write-once). The merge commit of `Prepare release vX.Y.Z` did not exist before the PR opened, so there is nothing to move — the tag is created here for the first time. A pre-existing tag at a different commit fails the step loudly rather than being force-moved (see [Tag immutability](./tag-immutability.md#tag-immutability)). The checkout uses `persist-credentials: false` so this push authenticates as the **CI app** (via `git remote set-url`), not the default `GITHUB_TOKEN` — a `GITHUB_TOKEN` push creates no workflow run, so a persisted-credential push would silently skip `tags.yaml` entirely and its stable-tag backstops (`generate-changelog`, `update-website-docs`) would never fire. +3. **Cut the `api/apps/v1alpha1/vX.Y.Z` Go-module tag** (write-once, stable only) at the same commit, so Go consumers of `api/apps/v1alpha1` get the release. Moved here from `tags.yaml`, whose `prepare-release` body is skipped for a promoted stable (the draft already exists). +4. **Ensure the maintenance branch `release-X.Y` exists** at the tag commit. Since the rc freeze creates it, this is normally a no-op fast-forward — the merge commit is already the branch tip. Still created if missing; updated fast-forward-only — a non-fast-forward update warns and is left for a maintainer rather than being force-updated. +5. **Publish the draft release**, with `docs/releasenotes/vX.Y.Z.md`, or without it `docs/changelogs/vX.Y.Z.md`, from the merge commit as its body (absent or whitespace-only files warn and leave the draft's body in place, rather than publishing blank notes, and a body over 100000 bytes is published as a link, see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). `make_latest` is computed against published-non-prerelease tags: prereleases stay `false`; tags older than the current max stay `false` (and the current max is force-restored to `latest` if necessary, so an older patch tag cut after a newer minor won't downgrade `latest`). +6. **Retag the verified packages candidate and rc container images to stable** by digest (`hack/promote-retag.sh`, no container rebuild) and **publish the stable `cozy-installer` chart**. `:latest` (on both the images and the chart) moves only when this release's `make_latest` was `true` — the same decision as step 5, so the release's `latest` and the images' `:latest` never disagree. The chart is packaged with `platformVersion` stamped into its default values so the documented `helm --install --version X.Y.Z` path reports the stable version. + +### Phase 6 — `update-releasenotes.yaml` (sync GitHub Release body) + +Fires on pushes to `main` that touch `docs/changelogs/v*.md`. Reads each `docs/changelogs/vX.Y.Z.md` and PATCHes the matching GitHub Release's `body` if it differs. Since Phase 5 now sets the body directly from the merge commit, this is no longer on the critical path — it covers later **edits** to a published changelog, the backstop PR landing a changelog that reached `main` after the fact, and manual re-syncs via `workflow_dispatch`. It only watches `main`, which is why finalize (not this job) is what populates a release promoted onto `release-X.Y` — since the freeze, that is every release rather than only a patch. + +The release body is `docs/releasenotes/vX.Y.Z.md` when it exists and is not blank, otherwise the changelog, which stays the full record either way, and a push to `main` touching either file fires this sync. Finalize sets the body after the stable tags exist, where a body GitHub refuses would strand the release, so at publish time a body over 100000 bytes is replaced by a one-line link to `docs/changelogs/vX.Y.Z.md` on the tag, with a warning in the run's log. This sync never writes that link: it warns and leaves such a release's body as it is. The `tags.yaml` backstop ports only the changelog to `main`, so release notes have to be added to `main` as well, otherwise the sync falls back to the changelog and replaces the release notes with it whenever the changelog fits. diff --git a/docs/release/cleanup.md b/docs/release/cleanup.md new file mode 100644 index 000000000..b192c5af5 --- /dev/null +++ b/docs/release/cleanup.md @@ -0,0 +1,6 @@ +## Cleanup after release + +- The `release-X.Y.Z` branch is deleted by GitHub when its PR merges. The `release-X.Y` maintenance branch is created by `cut-prerelease.yaml` at the rc freeze (create-only, never force-moved) and fast-forwarded afterwards by `pull-requests-release.yaml::Ensure maintenance branch`. +- The draft release is published by the same workflow. +- The release body is set by finalize from `docs/changelogs/vX.Y.Z.md` at publication. Afterwards, `update-releasenotes.yaml` re-syncs that file into the GitHub Release body on any push to `main` that touches it — so edits to a published changelog land on the release page the next time `main` moves. +- Local cleanup: remove your worktree (`git worktree remove`) and prune merged release branches. diff --git a/docs/release/introduction.md b/docs/release/introduction.md new file mode 100644 index 000000000..78c0e1781 --- /dev/null +++ b/docs/release/introduction.md @@ -0,0 +1,11 @@ +## Introduction + +Cozystack uses a staged release process to ensure stability and flexibility during development. + +There are three types of releases: + +- **Release Candidates (RC)** – Preview versions (e.g., `v1.2.0-rc.1`) used for final testing and validation. +- **Regular Releases** – Final versions (e.g., `v1.2.0`) that are feature-complete and thoroughly tested. +- **Patch Releases** – Bugfix-only updates (e.g., `v1.2.1`) made after a stable release, based on a dedicated release branch. + +All three are matched by `tags.yaml`'s regex `^v\d+\.\d+\.\d+(-(alpha|beta|rc)\.\d+)?$`. The tag push is the trigger for the whole pipeline. **Pre-release tags (`-alpha` / `-beta` / `-rc`) are cut with the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow** (manual dispatch from `main` or `release-X.Y`) — it pushes the tag as the CI app at the dispatch branch's tip, so GitHub fills `base_ref` (`tags.yaml`'s `Get base branch` step refuses tags pushed without a base) and the push triggers `tags.yaml` (a tag pushed with the default `GITHUB_TOKEN` would not). Stable `vX.Y.Z` tags are never pushed by hand at all — they are cut write-once by the promote flow at a PR merge commit. Only `-rc.N` pre-releases feed the promotion flow ([`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) accepts `vX.Y.Z-rc.N` alone); `-alpha.N` / `-beta.N` tags are built and published for download and validated on demand with the [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) button, but are never promoted to a stable release. diff --git a/docs/release/nightly-builds.md b/docs/release/nightly-builds.md new file mode 100644 index 000000000..df83de7f2 --- /dev/null +++ b/docs/release/nightly-builds.md @@ -0,0 +1,21 @@ +## Nightly builds + +A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs. + +[`nightly.yaml`](../../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished: + +1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`. +2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it. +3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply. + +No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package the current nightly publishes, named from the published `cozystack-packages:nightly` tree by [`hack/retention-packages.sh`](../../hack/retention-packages.sh), except packages linked to another repository, which are skipped with a warning; the floating `nightly`, release tags and untagged versions are never touched) and abandoned packages candidates. A candidate version is eligible only when all of these hold: it carries at least one tag and **every** tag on that manifest matches the strict `promotion-vX.Y.Z-from-vX.Y.Z-rc.N-run--` form, so a manifest that has been retagged stable is excluded; it was last updated more than 30 days ago; its digest is not pinned by `packages/core/installer/values.yaml` at the head of an **open** pull request that is same-repository, authored by `cozystack-ci[bot]`, on a `release-X.Y.Z` branch and carrying the `release` label; and its digest is not pinned by that same file at the tip of `main` or of any `release-X.Y` branch — which is what keeps a candidate safe between the promote PR merging and finalize retagging it. Any such PR head whose pin is not exactly one `digest=sha256:<64 hex>` stops the candidate sweep for that run and fails the job, without affecting the nightly sweep. + +Install a nightly: + +```bash +helm upgrade --install cozystack \ + oci://ghcr.io/cozystack/cozystack/cozy-installer --version 0.0.0-nightly.20260626.15992304129.1 +# ...or --version nightly for the latest +``` + +The matching Talos node image is `ghcr.io/cozystack/cozystack/cozystack-nocloud:0.0.0-nightly.20260626.15992304129.1` (pull with `oras`). diff --git a/docs/release/patch-releases.md b/docs/release/patch-releases.md new file mode 100644 index 000000000..9f23e23d6 --- /dev/null +++ b/docs/release/patch-releases.md @@ -0,0 +1,82 @@ +## Patch Releases + +A patch release now follows exactly the same shape as a [regular release](./regular-releases.md#regular-releases) — by the time `vX.Y.0` ships, everything is already happening on `release-X.Y`. The only difference is where the line starts: + +* `release-X.Y` already exists, created by the freeze at `vX.Y.0-rc.1`, so there is no branch to create. +* Patch commits are cherry-picked onto it, and the rc is cut by dispatching **from `release-X.Y`** — the same dispatch used for `rc.2` of the original release. + + +Let's assume that we've released `v1.2.0` and that development is ongoing. We have introduced a couple of new features and some fixes to features that we have released in `v1.2.0`. + +Once problems were found and fixed, a patch release is due. + +```mermaid +gitGraph + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" + checkout main + commit id: "feature 4" + commit id: "patch 1" + commit id: "feature 5" + commit id: "patch 2" +``` + + +1. Fixes reach the existing `release-1.2` branch from `main` through the backport batch, which cherry-picks what the [backport policy](./backports.md) admits and nothing else. + + The batch runs twice a week, so a fix merged into `main` reaches the line with the next batch that can pick it cleanly, without anyone asking for it. + + ```mermaid + gitGraph + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" + checkout main + commit id: "feature 4" + commit id: "patch 1" + commit id: "feature 5" + commit id: "patch 2" + checkout release-1.2 + cherry-pick id: "patch 1" + cherry-pick id: "patch 2" + ``` + + Merge the open batch pull request first. Before the cut, `./backport-batch check --line release-1.2 --cut` (build it with `go build ./cmd/backport-batch`) has to exit 0: it exits 1 while a fix the rules take is not on the line, including one that is only on the open batch pull request, which the cut would not carry, and prints the URLs; it prints what the batch deferred as warnings. See [Cherry-pick triage before a patch](./backport-mechanics.md#cherry-pick-triage-before-a-patch). + +2. The maintainer cuts a release candidate (`v1.2.1-rc.N`) via the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `release-1.2`**), which tags that branch's `HEAD`. CI builds and publishes the rc, pushes the `release-1.2.1-rc.N` staging branch, and runs the mandatory full `rc-e2e` job against the published tag. +3. Watch `rc-e2e` and wait for it to pass, then complete the other release checks. A successful manual `e2e-tag.yaml` run for `v1.2.1-rc.N` is also valid evidence if the latest `tags.yaml` run does not carry a green `rc-e2e`. **Optional:** press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.1.md` at rc time and bank it on the rc staging branch; promotion then reuses it instead of regenerating. +4. Once the checks are green, run [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) (manual dispatch **from `main`**, as for every promotion, not from `release-1.2`) with that rc tag. It runs the same five jobs as a regular promotion: the dispatch first gates on the latest tag-push `rc-e2e` or a successful tag-correlated manual E2E run (`skip_e2e_gate` overrides it), verifies that `release-1.2` already carries the candidate-aware pipeline, then commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning the candidate on the `release-1.2.1` branch, drafts the `v1.2.1` release with the restamped assets, generates or picks up `docs/changelogs/v1.2.1.md` and commits it onto that branch, opens the parked `cozystack/website` docs PR (updating the existing `v1.2/` directory in place, since a patch does not promote `next/`), and opens the promote PR into `release-1.2` with only the `release` label. The promote PR body carries the same ✅/⚠️ E2E, changelog, and website-docs lines and runs no E2E by default; add `full-e2e` manually to force a repeat. As with a regular release every stable name is deferred to merge; retention protects the temporary candidate while this PR is open and, after it merges, while `release-1.2` still pins it — until finalize retags it stable. + + ```mermaid + gitGraph + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" + checkout main + commit id: "feature 4" + commit id: "patch 1" + commit id: "feature 5" + commit id: "patch 2" + checkout release-1.2 + cherry-pick id: "patch 1" + cherry-pick id: "patch 2" tag: "v1.2.1-rc.1" + branch release-1.2.1 + commit id: "Prepare release v1.2.1" + checkout release-1.2 + merge release-1.2.1 id: "Pull request" + ``` + + Finally, when release is confirmed, the release sequence goes on. + +5. Maintainer reviews the PR against the [Phase 4 checklist](./ci-pipeline.md#phase-4--review-and-merge-the-promote-pr), confirms the E2E-verification and changelog lines and notes the parked website-docs line, and merges it — **do not squash-merge**. GitHub removes the merged branch `release-1.2.1` (auto-delete-on-merge is enabled). +6. CI workflow triggers on merge (all consumer-visible or write-once side effects, after the rc E2E gate and PR review): + 1. Verifies the pinned packages candidate against the merged tree and the original rc artifact's container digests. + 2. Creates the tag `v1.2.1` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. + 3. Cuts the write-once `api/apps/v1alpha1/v1.2.1` Go-module tag at the same commit. + 4. Publishes the release page (`draft` → `latest`; `latest` moves only if `v1.2.1` is the newest published stable). + 5. Retags the verified packages candidate and rc container images by digest to `v1.2.1` and publishes the stable `cozy-installer` chart — no container rebuild. +7. Once the release is published, merge the parked `cozystack/website` docs PR, then announce the release to the community. diff --git a/docs/release/pre-release-checklist.md b/docs/release/pre-release-checklist.md new file mode 100644 index 000000000..8e6257304 --- /dev/null +++ b/docs/release/pre-release-checklist.md @@ -0,0 +1,14 @@ +## Pre-release verification checklist + +For RCs and final releases, run this before merging the release PR. Each item is here because something has shipped without it and broken users. + +- [ ] **The published rc passed mandatory full E2E.** Confirm the promote PR says `✅ RC full e2e was verified`; the normal promote PR itself runs no E2E. Add `full-e2e` manually only when a repeat is required, and treat a ⚠️ bypass line as an explicit release-risk decision. +- [ ] **Upgrade from previous patch** (`vX.Y.(Z-1)`) on a real cluster. `helm upgrade` succeeds, all HelmReleases reach Ready, no CRD schema rejections. +- [ ] **Upgrade from previous minor** (`v(X-1).Y.0`) for RC and minor releases. Same assertions. +- [ ] **At least one tenant with `publishing.ingressName != tenant-root`** and a real Let's Encrypt-staging cert issuance. +- [ ] **Apply-mutate-reapply** on the kubernetes umbrella HelmRelease (catches second-reconcile bugs). +- [ ] **For every CRD-source bump** (kubevirt, cert-manager, flux, cilium, kamaji): `helm template | kubectl apply --dry-run=server` against the freshly-installed CRDs. +- [ ] **For every flux / helm-controller / kubevirt-operator / cert-manager bump:** paste the upstream breaking-changes section of the bump's CHANGELOG into the PR body before merge. +- [ ] **After cutting the tag,** watch `pull-requests-release.yaml::Finalize Release`. If it fails on `Draft release for v... not found`, undraft the release manually and file the workflow regression — this has been a known recurring failure since the `Publish draft release` step lost its explicit `github-token` in refactor `66a756b6`. +- [ ] **The promote PR carries `docs/changelogs/vX.Y.Z.md` and it was verified** per [Changelog pre-publish verification](./changelogs.md#changelog-pre-publish-verification). Finalize publishes this file as the release body, so verifying it before merge is verifying the release notes. +- [ ] **The `cozystack/website` docs PR stays parked until publish.** Promotion opens it early; merging it before the GitHub release for `vX.Y.Z` is live 404s the new version's docs and API links. Merge it right after finalize publishes the release (regular-release step 6, patch step 7). diff --git a/docs/release/regular-releases.md b/docs/release/regular-releases.md new file mode 100644 index 000000000..cdb007025 --- /dev/null +++ b/docs/release/regular-releases.md @@ -0,0 +1,78 @@ +## Regular Releases + +A regular release `vX.Y.0` is **promoted from a release-candidate that already passed e2e** — never tagged by hand and never rebuilt. The bytes shipped as `vX.Y.0` are the exact `vX.Y.0-rc.N` images, retagged by digest. We'll use `v1.2.0` as an example. + +```mermaid +gitGraph + commit id: "feature" + commit id: "feature 2" + commit id: "feature 3" tag: "v1.2.0-rc.1" +``` + +A regular release sequence runs as follows: + +1. Cut the first release candidate (`v1.2.0-rc.1`) from the last good commit on `main`: run the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `main`**) with the tag. It tags `main`'s HEAD as the CI app, then **creates `release-1.2` at that commit, freezing the line**; `tags.yaml` then builds the rc images, publishes the rc prerelease, pushes the digest-vendored `release-1.2.0-rc.N` staging branch, and runs its mandatory `rc-e2e` job by calling [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) against that published tag. +2. Watch the rc's mandatory full E2E and wait for it to pass, then complete the other checks in [Pre-release verification checklist](./pre-release-checklist.md#pre-release-verification-checklist). If the latest `tags.yaml` run cannot provide a green `rc-e2e`, a successful manual dispatch of [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) for the same tag is equivalent gate evidence. `main` is already open for the next minor, so anything landing there is **not** in this release until it reaches `release-1.2`: a fix merged into `main` gets there through the [backport batch](./backports.md#which-lines-the-batch-runs-for), which runs for `release-1.2` from the freeze on, and a fix with no counterpart on `main` through a maintainer's pull request against `release-1.2`. A later rc is cut by dispatching the workflow **from `release-1.2`**, and only once `./backport-batch check --line release-1.2 --cut` exits 0: merge the open batch pull request first, so that the rc carries every fix the policy takes. **Optional:** once the notes are worth reviewing, press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.0.md` now and bank it on the `release-1.2.0-rc.N` staging branch — so there is time to review and hand-edit it before promotion. Skipping the button changes nothing; promotion generates the changelog itself. + + ```mermaid + gitGraph + commit id: "feature" + commit id: "feature 2" + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout main + commit id: "feature 4 (next minor)" + checkout release-1.2 + commit id: "backported fix" tag: "v1.2.0-rc.2" + ``` + +3. Once the checks are green, run the [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) workflow (manual dispatch **from `main`**, the only branch it is dispatched from, whichever line the rc belongs to) with that rc tag. Its `parse` job verifies E2E before any staging: it accepts a successful `rc-e2e` job in the latest `tags.yaml` push run for the tag, or any retained successful manual `e2e-tag.yaml` run whose expanded E2E job name targets the tag. The emergency `skip_e2e_gate` input bypasses this check, logs a loud warning, and marks the promote PR with ⚠️. Before its first registry write, promotion also proves that the selected `main` or `release-X.Y` target already contains the candidate-aware PR gate, finalizer and trusted verifier. A maintenance branch that lacks them gets them before its next patch is promoted, through the batch with `backport/include` on the pull request that brought them to `main`, or through a maintainer's pull request against the line. Promotion **creates no stable name before merge**: dispatch does write to the registry — it publishes one `promotion-*` packages candidate, and that tag is real and publicly listable — but it creates no `vX.Y.Z` tag, moves no `latest`, and publishes no stable installer, so nothing a consumer resolves changes until the PR merges. It: + 1. Rewrites the rc version substring to `1.2.0` and commits that exact tree, serializes its `packages/` directory under a run-specific temporary OCI tag, then adds a second commit pinning the candidate's immutable digest in the installer before pushing `release-1.2.0`. Container digests remain exactly the rc's; the two commits give the OCI revision an honest source SHA without pretending the artifact can contain its own digest. + 2. Drafts the `v1.2.0` release and uploads the (restamped) assets. + 3. Generates `docs/changelogs/v1.2.0.md` (in a parallel job, detached at the rc tag) — or, if you pressed the rc-time `Generate RC Changelog` button, copies the file already banked on the `release-1.2.0-rc.N` staging branch instead of regenerating — and commits it onto the `release-1.2.0` branch, so the changelog is reviewed as part of the promotion and finalize can publish the release with it already in place. Non-blocking: if generation fails the PR still opens, carrying a ⚠️ line that says the changelog is missing. + 4. Opens the `cozystack/website` "update managed apps reference" PR (`update-docs-v1.2.0` branch), generating the docs from the `release-1.2.0` staging branch via `FETCH_REF`. Non-blocking, and **do not merge that PR until `v1.2.0` is published** — merging early breaks the live site's version routing (see [Phase 3](./ci-pipeline.md#phase-3--update-website-docs-now-the-backstop)). + 5. Opens the `chore(release): promote v1.2.0-rc.N -> v1.2.0` PR into `release-1.2`, labelled `release` only. The PR body records whether E2E was verified or bypassed, plus ✅/⚠️ changelog and website-docs lines. E2E does not run on this promote PR by default because the immutable rc already passed the mandatory full suite; a maintainer may add `full-e2e` to run it again. The stable retag itself is deferred to the merge (step 5), so an abandoned promotion leaves no stable-named images and cannot wedge a re-promotion; retention protects the temporary candidate for as long as this release PR remains open, then may remove it after 30 days. + + The PR targets the maintenance branch, not `main`: `promote-rc.yaml` prefers `release-X.Y` whenever it exists, and since the rc freeze it always does. The digest-vendored `Prepare release` commit therefore stays on the release line instead of landing on `main`. + + ```mermaid + gitGraph + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "backported fix" tag: "v1.2.0-rc.2" + branch release-1.2.0 + checkout release-1.2.0 + commit id: "Prepare release v1.2.0" + commit id: "Pin promoted packages artifact" + checkout release-1.2 + merge release-1.2.0 id: "Pull Request" + ``` + +4. Maintainer reviews the PR and the draft release against the [Phase 4 checklist](./ci-pipeline.md#phase-4--review-and-merge-the-promote-pr): confirm the E2E-verification line is ✅ (or explicitly accept its ⚠️ bypass), the changelog line is ✅, and note the website-docs line — the `cozystack/website` docs PR that promotion parked, which stays unmerged for now. E2E does not run on this promote PR by default. Then merge — **do not squash-merge**, the stable tag must attach to a real merge commit. GitHub removes the merged branch `release-1.2.0` (the repo has auto-delete-on-merge enabled). +5. CI workflow triggers on merge (this is where every consumer-visible or write-once side effect happens, after the rc E2E gate and PR review): + 1. Pulls the pinned packages candidate by digest and verifies that it matches the merged `packages/` tree apart from the artifact's impossible self-reference, carries no rc image tags, and preserves the original rc artifact's container `repo@digest` set. + 2. Creates the tag `v1.2.0` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. + 3. Cuts the write-once `api/apps/v1alpha1/v1.2.0` Go-module tag at the same commit. + 4. Ensures the `release-1.2` maintenance branch exists at the tag commit. Since the rc freeze created it, this is normally a no-op fast-forward: the merge commit is already its tip. + 5. Publishes the release page (`draft` → `latest`), using `docs/changelogs/v1.2.0.md` from the merge commit as the release body. + 6. Retags the verified packages candidate and rc container images by digest to `v1.2.0` (and `:latest` only when `v1.2.0` is the newest published stable), and publishes the stable `cozy-installer` chart — no container rebuild. +6. Once the release is published, merge the parked `cozystack/website` docs PR (opened back at promote time, step 3.iv) so the live site's version routing flips to `v1.2.0` — its versioned docs and `openapi.json` are on GitHub now. The maintainer can then announce the release to the community. + +```mermaid +gitGraph + commit id: "feature 2" + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout main + commit id: "feature 4 (next minor)" + checkout release-1.2 + commit id: "backported fix" tag: "v1.2.0-rc.2" + branch release-1.2.0 + checkout release-1.2.0 + commit id: "Prepare release v1.2.0" + checkout release-1.2 + merge release-1.2.0 id: "Release v1.2.0" tag: "v1.2.0" +``` + +Note that `main` never receives the release commit. It forked away at the freeze and carries the next minor's work. diff --git a/docs/release/release-candidates.md b/docs/release/release-candidates.md new file mode 100644 index 000000000..55f7eec61 --- /dev/null +++ b/docs/release/release-candidates.md @@ -0,0 +1,18 @@ +## Release Candidates + +Release candidates are Cozystack versions that introduce new features and are published before a stable release. Their purpose is to help validate stability before finalizing a new feature release. + +Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v1.2.0-rc.1`. The first one is cut from `main`, once all major features for the upcoming release have merged and the release enters its testing phase. + +**Cutting the first rc freezes the line.** [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` at the tagged commit, and from that point the release's content is closed: + +- Every later cut for that line — `rc.2`, `rc.3`, an `alpha`/`beta`, or a patch-line rc — must be dispatched from `release-X.Y`. A dispatch from `main` is refused. +- Fixes reach the release only through the backport batch onto `release-X.Y`, or a maintainer's pull request against it (see [Backports](./backports.md)). The batch runs for the line from its next scheduled run on, [while it has not shipped](./backports.md#which-lines-the-batch-runs-for): it takes the fixes merged into `main` since the freeze, and an older line takes each of them only once this line's batch that carries it has merged. +- A later rc is cut only once `backport-batch check --line release-X.Y --cut` exits 0, so that it carries every fix the batch takes: the open batch pull request merges first. +- `main` reopens immediately for the next minor, so feature work never has to wait for the release to ship. + +This is what makes an rc mean something. Previously `release-X.Y` was created only when the stable release merged, so `rc.2` was cut from `main`'s tip and silently absorbed everything that had landed since `rc.1` — the shipped release could contain code that no rc had ever validated. + +**Un-freezing is manual and deliberate.** If an `rc.1` turns out to be unusable and the line has to be re-cut from `main`, delete `release-X.Y` by hand first; the refuse step blocks every other route back to a `main` dispatch, which is the point of it. Nothing automates the deletion, and nothing should — throwing away a frozen line is a decision worth making explicitly. + +Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. After testing, if no critical issues remain, the last good rc is **promoted** to the regular release `vX.Y.0` — the exact rc images retagged by digest, never rebuilt and never tagged from a later commit (see [Regular Releases](./regular-releases.md#regular-releases)). Promotion cuts the write-once `vX.Y.0` tag at the promote PR's merge commit, fast-forwards the `release-X.Y` maintenance branch the freeze already created, and opens the way for patch releases. diff --git a/docs/release/release-time-fires.md b/docs/release/release-time-fires.md new file mode 100644 index 000000000..f58fd48b7 --- /dev/null +++ b/docs/release/release-time-fires.md @@ -0,0 +1,35 @@ +## Release-time fires — what's gone wrong recently + +These are real regressions that escaped to users. The pattern in each: code passed CI, shipped in a stable release, broke customers. Read them as a working list of pre-release verification gaps. + +### v1.3.0 — cert-manager ingressClassName regression (fix #2562) + +**Bug:** A commit migrated ACME HTTP-01 to use the modern `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The matching ingress-shim code that reads that annotation was added by cert-manager upstream PR #8244 and was **never backported** to the `release-1.19` branch — but cozystack 1.3.0 shipped cert-manager v1.19.3. The annotation was silently dropped at runtime and every solver Ingress fell back to the ClusterIssuer's default class `tenant-root`. Let's Encrypt validation broke for every tenant whose `publishing.ingressName != tenant-root`. + +**Blast radius:** v1.3.0, v1.3.1, and v1.1.7 (the migration was cherry-picked to `release-1.1`). + +**Why CI missed it:** the migration commit was self-contained and `helm template` rendered cleanly. There was no E2E that creates a tenant with a non-default ingress class and watches a real Let's Encrypt cert issue. Verification had been done from the docs side ("the API has been available since cert-manager 1.12"), not against the shipped cert-manager source. + +**Would have caught it:** non-root-tenant E2E with LE-staging cert issuance, asserting the solver Ingress lands on the tenant's class. + +### v1.4.0 — kubevirt-instancetypes upgrade failure (fix #2612) + +**Bug:** `packages/system/kubevirt-instancetypes/Makefile` contained `sed -i '/persistent: true/d' templates/preferences.yaml`, which stripped the only child of `preferredTPM`, leaving `preferredTPM:` (null) in six Windows preferences. KubeVirt v1.6.x silently accepted it. The v1.6.3 → v1.8.2 operator bump (2026-04-27) hardened the CRD's OpenAPI schema and started rejecting null for an object-typed field. `helm upgrade` then failed on every cluster with `kubevirt-instancetypes` enabled. + +**Why CI missed it:** the kubevirt-operator bump and the inert `preferences.yaml` were two weeks apart in commit history. CI only tests fresh installs against the new CRDs; it doesn't run `helm upgrade` from prior versions. Schema regression on a vendored asset that hadn't been touched in 18 months. + +**Would have caught it:** `helm template | kubectl apply --dry-run=server -f -` against the freshly-installed CRDs of every bumped operator. Or an upgrade-from-N-1 E2E lane. + +### v1.4.0 — Flux 2.7 → 2.8 readiness deadlock (#2602) + +**Bug:** helm-controller v1.5.0 (shipped in Flux 2.8) changes the default wait strategy to kstatus polling, which polls **every applied resource including child HelmReleases**. The `packages/apps/kubernetes` umbrella has 19 child HRs that each `dependsOn: parent` (with a `lookup` guard so the dependency only activates after the parent exists). Pre-2.8: parent went Ready as soon as Helm returned. Post-2.8: parent waits for children, children wait for parent. Deadlock. + +**Why CI missed it:** the cycle only manifests on the second reconcile (parent exists, so the `lookup` activates the child's `dependsOn`). Fresh-install E2E dodges this — the parent-already-exists branch isn't taken on the first apply. + +**Would have caught it:** apply-mutate-reapply E2E (any chart bump in the kubernetes umbrella triggers a second reconcile). Plus an explicit PR-template requirement: for any flux/helm-controller bump, paste the upstream breaking-changes section into the PR body. + +### Patterns these share + +1. **Fresh-install CI is insufficient.** All three regressions passed fresh-install lanes. +2. **Upstream breaking-defaults bumps are the common shape.** kubevirt CRD hardening, helm-controller default wait strategy, cert-manager API surface drift. Pre-merge reading of upstream changelogs would have caught two of three. +3. **One-engineer-bitten-once is the only safety net for live verification.** The cert-manager bump PR included an explicit live-cluster verification step — because the same engineer had been bitten the prior week. There is no checklist requiring it. diff --git a/docs/release/signatures-and-sbom.md b/docs/release/signatures-and-sbom.md new file mode 100644 index 000000000..8aa47051e --- /dev/null +++ b/docs/release/signatures-and-sbom.md @@ -0,0 +1,29 @@ +## Release signatures and SBOM + +A release cut from a tree that has [`hack/sign-release-assets.sh`](../../hack/sign-release-assets.sh) carries three files beside its other assets, which [`hack/upload-assets.sh`](../../hack/upload-assets.sh) has that script write before it uploads anything. Releases cut before it have none of them, and neither does a maintenance line until the script is backported to it (see below). + +- `cozystack-source-sbom.spdx.json` is an SPDX SBOM, made with syft, of the committed source tree (the Go modules, npm lockfiles, GitHub Actions references and other manifests syft reads in it) and of the `cozypkg` binaries in the `cozypkg-*.tar.gz` assets. It does not describe what runs in the cluster: the container images carry their own SBOMs, see [SBOM of the container images](#sbom-of-the-container-images), and the Talos installer image, ISO, disk, kernel and initramfs images have none. Its name says what it covers, so that `cozystack-sbom` stays free for an SBOM of the release as a whole. +- `cozystack-checksums.txt` holds the SHA-256 of every other asset and of the SBOM. +- `cozystack-checksums.txt.sigstore.json` is a Sigstore bundle with a keyless cosign signature over the checksums. Its certificate names the workflow that published the release: `tags.yaml` for a pre-release, `promote-rc.yaml` for a stable one. + +One signature covers every asset through the checksums. To check a download, verify the checksums, then the file against them, with cosign v3.1.3 or later (v2.6.5 or later on the v2 line): + +```sh +cosign verify-blob cozystack-checksums.txt \ + --bundle cozystack-checksums.txt.sigstore.json \ + --certificate-identity-regexp '^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com && + sha256sum --check --ignore-missing cozystack-checksums.txt +``` + +Older cosign accepts a legacy-format bundle that carries a bare public key in place of a certificate and skips the identity and issuer checks for it ([GHSA-fx35-mq7g-6g98](https://github.com/sigstore/cosign/security/advisories/GHSA-fx35-mq7g-6g98)), so a bundle swapped in next to the download, by a mirror or a proxy, passes there. The bundles this repository writes are in the new format, but cosign picks the format from the file, and `--new-bundle-format` does not stop that on the v3 line (v3.1.2 still accepts such a bundle with it), so the cosign version is what protects the check. + +The regexp pins the ref each workflow runs from: `tags.yaml` on a `v*` tag push, and `promote-rc.yaml` dispatched from `main`, the only branch it is dispatched from. A passing check proves that the checksums were signed by one of those runs in `cozystack/cozystack`. A `promote-rc.yaml` dispatched from another branch fails this check in its own upload step, with no asset uploaded, and an edited `promote-rc.yaml` has to be merged into `main` before it can sign. Until the `v*` [tag rulesets](./tag-immutability.md#repo-side-enforcement-tag-protection-rulesets) restrict tag creation to the CI app, though, anyone with write access can push a `v*` tag whose tree carries an edited `tags.yaml`, and its signature passes. + +The script runs the same `cosign verify-blob` before it returns, so a signature this check would reject fails the job before the first upload, and the release is not published with assets but without a signature. Both jobs that call `make upload_assets`, `tags.yaml::prepare-release` and `promote-rc.yaml::promote`, hold `id-token: write` for the signing identity and install syft and cosign v3.1.3, the version users are asked for above. They install syft from its release tarball, checked against a SHA-256 committed in the workflow, and not with `anchore/sbom-action/download-syft`, which runs an install script fetched at run time in a job that holds the release credentials; [`hack/ci-tool-installs-contract.bats`](../../hack/ci-tool-installs-contract.bats) keeps it that way. A stable release gets its own three files rather than its rc's: promotion restamps several assets, so the checksums are recomputed and signed again. + +`promote-rc.yaml` runs the `hack/upload-assets.sh` of the rc tree, not the dispatch ref's. A stable release is therefore signed only when its rc was tagged from a tree that has `hack/sign-release-assets.sh`, and a maintenance line starts carrying signatures once that script is backported to its `release-X.Y` branch. + +### SBOM of the container images + +The container images a release pins by digest each carry an SPDX SBOM of the packages in them, which BuildKit attaches at build time as an attestation (`SBOM=1` in `tags.yaml`, scanner and flags in [`hack/common-envs.mk`](../../hack/common-envs.mk)). The attestation is an entry of the image's index, so the digest the release pins names both, and promotion copies it with the image. Read it with `docker buildx imagetools inspect @ --format '{{ json .SBOM }}'`, which prints one document per platform. The attestations are not signed. A release cut before `SBOM=1` was set has none, and neither has the Talos installer image, which the imager builds and `skopeo` pushes, nor the third-party images `packages/` vendors by digest. diff --git a/docs/release/splitting-bundle-prs.md b/docs/release/splitting-bundle-prs.md new file mode 100644 index 000000000..ace989a88 --- /dev/null +++ b/docs/release/splitting-bundle-prs.md @@ -0,0 +1,10 @@ +## Splitting a release-blocking bundle PR + +Sometimes the work that has to land before a release is a 40-commit grab bag (CI stabilization, dependency bumps, chart fixes). Splitting it makes it reviewable and survivable. The strategy that has worked: + +1. **Verify what's already on `main`.** Use `git merge-base --is-ancestor` and subject grep to drop commits that have already landed. +2. **Find existing open PRs that cover the same commits.** Force-push to the existing branch instead of opening a duplicate — preserves review threads. +3. **Identify the "bottleneck" PR.** The one whose changes make previously-silent misconfigurations into hard errors (typically an operator-version bump). Everything else can fan out in parallel. +4. **Squash iterative same-author bug-on-bug fixes** inside each split. Preserve authorship via `--author` on the squashed commit. +5. **Mark the keystone PR as draft** and link in the body to the dependency PRs. Reviewers know it intentionally fails CI until the chain lands. +6. **Beware of concurrent registry pushes.** When fanning out 5+ PR CI runs simultaneously, multiple `make image` jobs can collide on shared floating tags. The `IMAGE_TAG=pr--` refactor solved this; if you see `Conflicted with another upload of the same manifest`, `gh run rerun --failed`. diff --git a/docs/release/stable-tags.md b/docs/release/stable-tags.md new file mode 100644 index 000000000..af908d046 --- /dev/null +++ b/docs/release/stable-tags.md @@ -0,0 +1,7 @@ +## Stable tags come from rc promotion + +A stable `vX.Y.Z` is created only by **promoting an existing release-candidate** that already passed e2e — never by rebuilding and never by a cron. + +- [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) is triggered manually once an rc has gone green. It rewrites the rc version substring in the vendored image tags (the container digests stay the rc's), commits and publishes the resulting declarative packages tree under a unique run-specific `promotion-*` tag, commits that digest pin, and opens the `release-X.Y.Z` staging PR. The PR gate verifies the candidate against the prospective merge and original rc artifact; finalize repeats that check before creating the write-once stable tag at the merge commit, then retags the verified packages artifact and rc's already-built, e2e-passed container digests to the stable tag **by digest** (no container rebuild — see [`hack/promote-retag.sh`](../../hack/promote-retag.sh)) and publishes the release. An abandoned promotion leaves no stable-named bytes; its temporary candidate is retention-managed. + +So a commit on a supported `release-X.Y` line ships when a maintainer promotes the next rc for that line — not automatically within 24h. Pushing a stable `vX.Y.Z` tag by hand is **not** a supported path: `tags.yaml` fails fast on a stable tag that has no pre-existing draft ("stable tags come from promote-rc.yaml"), and even so the finalize step would refuse to move a pre-existing tag. Pre-releases (`vX.Y.Z-rc.N`, `-alpha.N`, `-beta.N`) are cut with the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow, which pushes the tag as the CI app; [`tags.yaml`](../../.github/workflows/tags.yaml) then fires on that push, builds it, and publishes the pre-release. Cutting via the workflow (rather than a manual `git push`) is what lets repo admins lock `v*` tag creation to the CI app — see the tag-protection note below. diff --git a/docs/release/tag-immutability.md b/docs/release/tag-immutability.md new file mode 100644 index 000000000..fe7418279 --- /dev/null +++ b/docs/release/tag-immutability.md @@ -0,0 +1,18 @@ +## Tag immutability + +Published tags are **write-once** — once a `vX.Y.Z` or rc tag is pushed it is never moved or deleted. (Nightlies are not git tags at all — they are GHCR OCI tags, see [Nightly builds](./nightly-builds.md#nightly-builds).) Moving a tag silently poisons the Go module proxy / pkg.go.dev cache for `api/apps/v1alpha1/vX.Y.Z` and drifts SBOM/provenance toolchains, so the release flow is built so a move is impossible by construction: + +| File | Tag / branch handling | +|------|------------------------| +| [`pull-requests-release.yaml`](../../.github/workflows/pull-requests-release.yaml) | Verifies the pinned packages candidate before creating a stable name. Creates `vX.Y.Z` at the PR merge commit **write-once** (create if absent, no-op if unchanged, fail loudly if it would move — the merge commit is new, so there is nothing to move). Cuts the `api/apps/v1alpha1/` Go-submodule tag write-once at the same commit, **stable only** (never rc/beta/alpha). The `release-X.Y` maintenance branch is fast-forward-only. Retags the verified packages artifact and rc container digests to the stable image tag (also write-once at the image level). | +| [`tags.yaml`](../../.github/workflows/tags.yaml) | The `release-X.Y.Z-rc.N` staging branch is a mutable staging ref (compare-before-force: skipped when unchanged, force+log only when it genuinely moves). Fails fast on a stable `vX.Y.Z` tag with no pre-existing draft — stable tags come from `promote-rc.yaml`, never a hand push. | +| [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) | Mutates no stable tags. Before staging, verifies green full E2E from the latest rc tag-push run or a tag-correlated manual published-tag run (unless loudly bypassed). Commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning its digest on the `release-X.Y.Z` branch, generates the changelog there, and opens the promote PR with `release` but no automatic `full-e2e`; the PR gate verifies the candidate and finalize repeats that verification before the stable tag and image retag. The staging branch is force-pushed on re-dispatch, but an existing `docs/changelogs/vX.Y.Z.md` is preserved across the rebuild. | + +This is the immutable-tag + rc-promotion model from [#2677](https://github.com/cozystack/cozystack/issues/2677): stable `vX.Y.Z` is created only by promoting an existing rc, rc tags are write-once, and `api/apps/v1alpha1/vX.Y.Z` is created only on a stable release. The old nightly `auto-release.yaml` (which delete-recreated auto-bumped patch tags) has been removed. rc and stable tags accrete permanently; the only churning artifacts are GHCR nightlies and abandoned temporary promotion candidates, which [`retention.yaml`](../../.github/workflows/retention.yaml) prunes (see [Nightly builds](./nightly-builds.md#nightly-builds)) — GHCR storage is the cost vector to watch. + +### Repo-side enforcement (tag-protection rulesets) + +The guarantees above are enforced in the workflows. Repo admins back them with two GitHub **tag rulesets** targeting `v*`: + +- **Immutability** — *Restrict updates* + *Restrict deletions*, **no bypass**. No actor (not even the CI app) can move or delete a shipped `v*` tag. Safe to enable at any time: nothing in the release flow ever moves or deletes a `v*` tag, it only creates new ones. +- **Creation control** — *Restrict creations* with the **CI app as the sole bypass actor**, plus *Limit branches/tags updated in a single push* = 1. This turns "don't hand-push tags" from convention into hard enforcement: every `v*` tag — pre-release **and** stable — can then be created only by CI. Enable this **only after** [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) is in use, since that workflow is what makes CI the sole creator of pre-release tags; enabling it earlier would lock maintainers out of cutting rc/alpha/beta. (The Go-module tag `api/apps/v1alpha1/*` gets an equivalent pair of rulesets.) diff --git a/docs/security/incident-response.md b/docs/security/incident-response.md index 62469519e..156ce4f2f 100644 --- a/docs/security/incident-response.md +++ b/docs/security/incident-response.md @@ -40,11 +40,7 @@ One person may hold more than one role on a small incident. 4. **Private fix development.** The fix is developed under a **GitHub Security Advisory** (private fork) so that code and discussion stay confidential until release. -5. **Coordinated release.** The fix is shipped through the normal patch-release - and backport process. Which supported lines receive the backport is decided - per fix, weighing severity against backport risk — see the *Supported - Versions* table in `SECURITY.md`, which also records that no documented rule - governs that choice. +5. **Coordinated release.** The fix is shipped through the normal patch-release and backport process: the backport batch carries it onto the supported lines, the two latest minor releases, newest first, by the rules in [`docs/release/backports.md`](../release/backports.md), which a maintainer can override for a single fix. See the *Supported Versions* table in `SECURITY.md`. 6. **Disclosure.** A GHSA — and, when applicable, a CVE — is published. Public disclosure follows the outbound channels: GitHub Releases and release notes, the changelog, and the GHSA itself. The reporter is credited unless anonymity diff --git a/examples/backups/kafka/92-scenario-user-restore.md b/examples/backups/kafka/92-scenario-user-restore.md index 3215fe056..d637a6c9f 100644 --- a/examples/backups/kafka/92-scenario-user-restore.md +++ b/examples/backups/kafka/92-scenario-user-restore.md @@ -71,9 +71,6 @@ The script: had already replayed. The Pod log names each topic before its replay and again once it landed; empty or delete every topic it shows as started, not only the one the refusal names, then re-run. -- The tarball in object storage is not managed by the Cozystack `Backup` - lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 - object in place. Tenants who need to reclaim space should delete the object - (or set a bucket lifecycle policy). +- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. A Simple or MultiZone Cozystack SeaweedFS applies such rules once a day only while its `lifecycle.enabled` is on, which is not the default; a Client instance applies none itself, so the SeaweedFS it connects to must have `lifecycle.enabled` on. Another object store applies them only if it implements bucket lifecycle. - Consumer-group offsets, topic configs, ACLs and users are not part of the backup; consumers start from the restored topic's fresh offsets. diff --git a/examples/backups/nats/92-scenario-user-restore.md b/examples/backups/nats/92-scenario-user-restore.md index 9f0e7c02a..5aefface0 100644 --- a/examples/backups/nats/92-scenario-user-restore.md +++ b/examples/backups/nats/92-scenario-user-restore.md @@ -55,9 +55,6 @@ The script: - Both flows rely on the strategy template branching on `.Mode`. See `01-create-strategy.sh`. -- The tarball in object storage is not managed by the Cozystack `Backup` - lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 - object in place. Tenants who need to reclaim space should delete the object - (or set a bucket lifecycle policy). +- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. A Simple or MultiZone Cozystack SeaweedFS applies such rules once a day only while its `lifecycle.enabled` is on, which is not the default; a Client instance applies none itself, so the SeaweedFS it connects to must have `lifecycle.enabled` on. Another object store applies them only if it implements bucket lifecycle. - `nats stream restore` recreates the stream's configuration from the snapshot; consumers are restored as captured at backup time. diff --git a/go.mod b/go.mod index 34352e583..cb49f9f24 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/google/cel-go v0.29.2 github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 @@ -83,7 +84,6 @@ require ( github.com/go-openapi/testify/enable/yaml/v2 v2.8.0 // indirect github.com/go-openapi/testify/v2 v2.8.0 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe // indirect diff --git a/hack/backport-batch-scheduled.bats b/hack/backport-batch-scheduled.bats new file mode 100644 index 000000000..e844869e8 --- /dev/null +++ b/hack/backport-batch-scheduled.bats @@ -0,0 +1,186 @@ +#!/usr/bin/env bats +# Tests for hack/backport-batch-scheduled.sh, the scheduled backport batch, and +# for the workflow that runs it, .github/workflows/backport-batch.yaml. +# +# The backport-batch binary is stubbed through BACKPORT_BATCH: every call is +# recorded with the tokens gh would see, `lines` answers with $STUB_LINES, and +# a subcommand fails for the line named in $STUB_FAIL_. +# +# Run with: bats hack/backport-batch-scheduled.bats + +load test_helper + +SCRIPT="${BATS_TEST_DIRNAME}/backport-batch-scheduled.sh" +WF="${BATS_TEST_DIRNAME}/../.github/workflows/backport-batch.yaml" + +setup() { + strict_setup + export TMPDIR="$BATS_TEST_TMPDIR" + export CALLS="${BATS_TEST_TMPDIR}/calls" + export BACKPORT_BATCH="${BATS_TEST_TMPDIR}/backport-batch" + cat > "$BACKPORT_BATCH" <<'EOF' +#!/usr/bin/env bash +printf 'GH_TOKEN=%s GITHUB_TOKEN=%s %s\n' "${GH_TOKEN-unset}" "${GITHUB_TOKEN-unset}" "$*" >> "$CALLS" +line=${3:-} +case "$1" in + lines) printf '%s\n' $STUB_LINES ;; + run) [ "${STUB_FAIL_run:-}" != "$line" ] || exit 2 ;; + check) [ "${STUB_FAIL_check:-}" != "$line" ] || exit 1 ;; + notify) + [ "${STUB_FAIL_notify:-}" != "$line" ] || exit 2 + echo "$line: something waits" + ;; +esac +EOF + chmod +x "$BACKPORT_BATCH" + export STUB_LINES="release-1.7 release-1.6" + unset STUB_FAIL_run STUB_FAIL_check STUB_FAIL_notify + unset BACKPORT_ALERT_TEAM GITHUB_STEP_SUMMARY GH_TOKEN + export GITHUB_TOKEN=job-token + export BACKPORT_BATCH_TOKEN=batch-token + : > "$CALLS" +} + +# The calls recorded, without the token fields. +calls() { + sed 's/^GH_TOKEN=[^ ]* GITHUB_TOKEN=[^ ]* //' "$CALLS" +} + +@test "runs, checks and reports each line, newest first, pushing with the batch's token alone" { + run "$SCRIPT" + [ "$status" -eq 0 ] + [ "$(calls)" = "lines +run --line release-1.7 --continue --push +check --line release-1.7 +notify --line release-1.7 --push +run --line release-1.6 --continue --push +check --line release-1.6 +notify --line release-1.6 --push" ] + # Every call sees the batch's token, and the job's token is gone. + [ "$(grep -c '^GH_TOKEN=batch-token GITHUB_TOKEN=unset ' "$CALLS")" -eq 7 ] +} + +@test "a dry run pushes, opens and comments nothing, and needs no token" { + unset BACKPORT_BATCH_TOKEN + run "$SCRIPT" --dry-run + [ "$status" -eq 0 ] + if grep -q -- '--push' "$CALLS"; then + echo "a dry run passed --push:"; cat "$CALLS" + false + fi + [ "$(calls | grep -c '^run --line release-.* --continue$')" -eq 2 ] + [ "$(calls | grep -c '^notify --line release-.*[0-9]$')" -eq 2 ] +} + +@test "--no-workflow-files reaches run alone, and an unknown argument stops the script" { + run "$SCRIPT" --no-workflow-files + [ "$status" -eq 0 ] + [ "$(calls | grep -c '^run --line release-.* --continue --no-workflow-files --push$')" -eq 2 ] + if calls | grep -v '^run ' | grep -q -- '--no-workflow-files'; then + echo "the flag reached another command:"; calls + false + fi + + : > "$CALLS" + run "$SCRIPT" --dry-run --no-workflow-files + [ "$status" -eq 0 ] + [ "$(calls | grep -c '^run --line release-.* --continue --no-workflow-files$')" -eq 2 ] + + : > "$CALLS" + run "$SCRIPT" --no-workflows + [ "$status" -eq 2 ] + [ ! -s "$CALLS" ] +} + +@test "without BACKPORT_BATCH_TOKEN it stops before anything, even with the job's token" { + unset BACKPORT_BATCH_TOKEN + run "$SCRIPT" + [ "$status" -eq 2 ] + [[ "$output" == *"BACKPORT_BATCH_TOKEN is empty"* ]] + [ ! -s "$CALLS" ] +} + +@test "outside a worktree of the repository it stops before anything" { + export GIT_CEILING_DIRECTORIES="$BATS_TEST_TMPDIR" + mkdir "$BATS_TEST_TMPDIR/elsewhere" + cd "$BATS_TEST_TMPDIR/elsewhere" + run "$SCRIPT" + [ "$status" -eq 2 ] + [[ "$output" == *"not in a worktree of the repository"* ]] + [ ! -s "$CALLS" ] +} + +@test "the team is mentioned only when BACKPORT_ALERT_TEAM is set" { + export BACKPORT_ALERT_TEAM=cozystack/maintainers + run "$SCRIPT" + [ "$status" -eq 0 ] + [ "$(calls | grep -c '^notify --line release-.* --mention cozystack/maintainers --push$')" -eq 2 ] + + export BACKPORT_ALERT_TEAM= + : > "$CALLS" + run "$SCRIPT" + [ "$status" -eq 0 ] + if grep -q -- '--mention' "$CALLS"; then + echo "an empty team was mentioned:"; cat "$CALLS" + false + fi +} + +@test "a line that fails goes into the summary, the other line still runs, and the job fails" { + export GITHUB_STEP_SUMMARY="${BATS_TEST_TMPDIR}/summary" + export STUB_FAIL_run=release-1.7 STUB_FAIL_check=release-1.6 + run "$SCRIPT" + [ "$status" -eq 1 ] + [ "$(calls | grep -c '^run --line release-1.6 ')" -eq 1 ] + grep -qx -- '- release-1.7: the run failed with exit 2' "$GITHUB_STEP_SUMMARY" + grep -qx -- '- release-1.6: the check exited 1; see its output above' "$GITHUB_STEP_SUMMARY" + grep -qx -- '- release-1.7: something waits' "$GITHUB_STEP_SUMMARY" + + # Each failure fails the job on its own. + export STUB_FAIL_run=release-1.7 STUB_FAIL_check= + run "$SCRIPT" + [ "$status" -eq 1 ] + export STUB_FAIL_run= STUB_FAIL_check=release-1.6 + run "$SCRIPT" + [ "$status" -eq 1 ] + export STUB_FAIL_check= STUB_FAIL_notify=release-1.6 + run "$SCRIPT" + [ "$status" -eq 1 ] + export STUB_FAIL_notify= + run "$SCRIPT" + [ "$status" -eq 0 ] +} + +@test "the workflow runs Monday and Thursday as the cozystack-ci App and never with the job's token" { + [ "$(yq -r '.on.schedule[0].cron' "$WF")" = "23 6 * * 1,4" ] + [ "$(yq -r '.on.schedule | length' "$WF")" -eq 1 ] + [ "$(yq -o=json -I=0 '.permissions' "$WF")" = "{}" ] + [ "$(yq -o=json -I=0 '.jobs.batch.permissions' "$WF")" = '{"contents":"read"}' ] + [ "$(yq -r '.jobs.batch.steps[] | select(.uses == "actions/checkout*") | .with."persist-credentials"' "$WF")" = "false" ] + run_step='.jobs.batch.steps[] | select(.name == "Run the batch")' + [ "$(yq -r "$run_step | .run" "$WF")" = "hack/backport-batch-scheduled.sh --no-workflow-files" ] + [ "$(yq -r "$run_step | .env.BACKPORT_BATCH_TOKEN" "$WF")" = '${{ steps.app-token.outputs.token }}' ] + [ "$(yq -r "$run_step | .env.BACKPORT_BATCH_BOT" "$WF")" = '${{ steps.app-token.outputs.app-slug }}[bot]' ] + + # The App token covers this repository alone, with Contents and Pull + # requests and nothing else: no workflows, no packages. + mint='.jobs.batch.steps[] | select(.id == "app-token")' + [ "$(yq -r "$mint | .uses" "$WF")" = "actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547" ] + [ "$(yq -r "$mint | .with.owner" "$WF")" = "cozystack" ] + [ "$(yq -r "$mint | .with.repositories" "$WF")" = "cozystack" ] + [ "$(yq -r "$mint | .with | to_entries | map(select(.key | test(\"^permission-\"))) | map(.key + \"=\" + .value) | join(\",\")" "$WF")" = "permission-contents=write,permission-pull-requests=write" ] + # A token minted after the checkout lives an hour; the job ends before. + [ "$(yq -r '.jobs.batch."timeout-minutes"' "$WF")" -le 45 ] + + # The commits carry the App's bot, read from the token, not a person. + identity='.jobs.batch.steps[] | select(.name == "Use the App'"'"'s bot as git identity and credential")' + [ "$(yq -r "$identity | .env.APP_SLUG" "$WF")" = '${{ steps.app-token.outputs.app-slug }}' ] + yq -r "$identity | .run" "$WF" | grep -qF 'gh api "/users/${APP_SLUG}[bot]" --jq .id' + yq -r "$identity | .run" "$WF" | grep -qF 'user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com"' + + # No step reaches for the job's own token or a person's batch token. + if grep -v '^[[:space:]]*#' "$WF" | grep -Eq 'secrets\.GITHUB_TOKEN|github\.token|secrets\.BACKPORT_BATCH_TOKEN|gh api user'; then + echo "the workflow reads the job's own token or a person's" + false + fi +} diff --git a/hack/backport-batch-scheduled.sh b/hack/backport-batch-scheduled.sh new file mode 100755 index 000000000..ddc7f886d --- /dev/null +++ b/hack/backport-batch-scheduled.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# The scheduled backport batch (docs/release/backports.md): for each line the +# batch runs for, newest first, continue the open batch pull request or open +# one, check the line, and report what waits on the batch. This is what +# .github/workflows/backport-batch.yaml runs; a maintainer can run it by hand +# from a clean worktree of the repository. +# +# Usage: backport-batch-scheduled.sh [--dry-run] [--no-workflow-files] +# +# --dry-run builds and checks each batch in this worktree and prints what it +# would report, but pushes nothing, opens or edits no pull request and writes +# no comment. Its check sees the line without the batch it just built, so it +# can exit 1 where the real run's check passes. Without --dry-run the batch's +# token has to be in BACKPORT_BATCH_TOKEN, and it is the only token gh sees: +# the job's own GITHUB_TOKEN is never a fallback, because a push made with it +# starts no CI on the batch pull request. The workflow passes the cozystack-ci +# App's token; a maintainer running this by hand passes their own. +# +# --no-workflow-files goes to run: an original that changes .github/workflows/ +# is left on the batch pull request for a maintainer instead of picked, for a +# token that may not push such a commit, as the workflow's is. +# +# Environment: +# BACKPORT_BATCH the backport-batch binary (default: built from this tree) +# BACKPORT_BATCH_TOKEN the token the batch pushes, opens and comments with +# BACKPORT_BATCH_BOT the bot the scheduled batch writes as, [bot], +# whose batch pull request and comment are trusted; +# cozystack-ci[bot] when unset +# BACKPORT_ALERT_TEAM the team or person the batch's comment mentions, +# org/team or a login; nobody is mentioned when it is +# unset +# GITHUB_STEP_SUMMARY when set, each finding is appended to it as a line +# +# Exits 0 when every line ran and its check passed, 1 when a line's run, +# check or report failed, and 2 on a usage or setup error. +set -euo pipefail + +die() { + echo "backport-batch-scheduled: $*" >&2 + exit 2 +} + +usage="usage: backport-batch-scheduled.sh [--dry-run] [--no-workflow-files]" +dry_run=false +# An empty array expands under set -u only as ${a[@]+"${a[@]}"} on bash before +# 4.4, which is what macOS ships. +workflows=() +for arg in "$@"; do + case "$arg" in + --dry-run) dry_run=true ;; + --no-workflow-files) workflows=(--no-workflow-files) ;; + *) die "$usage" ;; + esac +done + +push=() +if ! $dry_run; then + [ -n "${BACKPORT_BATCH_TOKEN:-}" ] || + die "BACKPORT_BATCH_TOKEN is empty; the workflow sets it to the cozystack-ci App's token, a run by hand to your own (docs/release/backports.md#the-scheduled-run). The job's GITHUB_TOKEN is not used: its pushes start no CI." + export GH_TOKEN="$BACKPORT_BATCH_TOKEN" + unset GITHUB_TOKEN + push=(--push) +fi + +mention=() +if [ -n "${BACKPORT_ALERT_TEAM:-}" ]; then + mention=(--mention "$BACKPORT_ALERT_TEAM") +fi + +top=$(git rev-parse --show-toplevel) || die "not in a worktree of the repository" +cd "$top" +out=$(mktemp -d) +bin=${BACKPORT_BATCH:-} +if [ -z "$bin" ]; then + # Built before the first run switches this tree to a batch branch, whose + # line may not have the tool at all. + bin="$out/backport-batch" + go build -o "$bin" ./cmd/backport-batch +fi + +report() { + echo "$1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf -- '- %s\n' "$1" >> "$GITHUB_STEP_SUMMARY" + fi +} + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '## Backport batch\n\n' >> "$GITHUB_STEP_SUMMARY" +fi +lines=$("$bin" lines) || die "cannot list the release lines" +[ -n "$lines" ] || die "no release line to run the batch for" + +status=0 +for line in $lines; do + echo "::group::$line" + rc=0 + "$bin" run --line "$line" --continue ${workflows[@]+"${workflows[@]}"} ${push[@]+"${push[@]}"} > "$out/$line.md" || rc=$? + if [ "$rc" -ne 0 ]; then + report "$line: the run failed with exit $rc" + status=1 + elif $dry_run; then + report "$line: the batch is built in this tree, its body in $out/$line.md" + fi + rc=0 + "$bin" check --line "$line" || rc=$? + if [ "$rc" -ne 0 ]; then + report "$line: the check exited $rc; see its output above" + status=1 + fi + rc=0 + notes=$("$bin" notify --line "$line" ${mention[@]+"${mention[@]}"} ${push[@]+"${push[@]}"}) || rc=$? + while IFS= read -r note; do + [ -z "$note" ] || report "$note" + done <<< "$notes" + if [ "$rc" -ne 0 ]; then + report "$line: reporting what waits on the batch failed with exit $rc" + status=1 + fi + echo "::endgroup::" +done +exit "$status" diff --git a/hack/backport-batch-verify-job.bats b/hack/backport-batch-verify-job.bats new file mode 100644 index 000000000..5cc0c1597 --- /dev/null +++ b/hack/backport-batch-verify-job.bats @@ -0,0 +1,59 @@ +#!/usr/bin/env bats +# Contract for the backport-batch-verify job of .github/workflows/pr-labeler.yaml, +# which runs `backport-batch verify` on every push to a batch pull request. +# +# The job runs under pull_request_target, so it holds a token on a pull +# request's events and must never run code from the pull request. These tests +# pin what keeps it that way: the tool is built from main, the head is only +# fetched, the token reads contents and nothing more, and no expression +# reaches a shell. +# +# Run with: bats hack/backport-batch-verify-job.bats + +load test_helper + +WF="${BATS_TEST_DIRNAME}/../.github/workflows/pr-labeler.yaml" +JOB='.jobs.backport-batch-verify' + +@test "the job runs on pull_request_target, so main's copy reaches release lines" { + [ "$(yq -r '.on | keys | .[]' "$WF")" = "pull_request_target" ] + [ "$(yq -r "$JOB.name" "$WF")" = "Backport batch shape" ] +} + +@test "the job runs only for a batch branch of this repository into a release line" { + cond="$(yq -r "$JOB.if" "$WF")" + [[ "$cond" == *"startsWith(github.event.pull_request.base.ref, 'release-')"* ]] + [[ "$cond" == *"startsWith(github.event.pull_request.head.ref, 'backport-batch/')"* ]] + [[ "$cond" == *"github.event.pull_request.head.repo.full_name == github.repository"* ]] + [ "$(grep -o '&&' <<<"$cond" | wc -l)" -eq 2 ] +} + +@test "the token reads contents and nothing else" { + [ "$(yq -o=json -I=0 "$JOB.permissions" "$WF")" = '{"contents":"read"}' ] +} + +@test "main is checked out, without credentials, and the head only fetched" { + [ "$(yq -r "$JOB.steps[0].uses" "$WF")" = "actions/checkout@11d5960a326750d5838078e36cf38b85af677262" ] + [ "$(yq -r "$JOB.steps[0].with.ref" "$WF")" = "main" ] + [ "$(yq -r "$JOB.steps[0].with.persist-credentials" "$WF")" = "false" ] + [ "$(yq -r "[$JOB.steps[] | select(.uses == \"actions/checkout*\")] | length" "$WF")" -eq 1 ] + [ "$(yq -r "$JOB.steps[] | select(.name == \"Build backport-batch from main\") | .run" "$WF")" = 'go build -o "$RUNNER_TEMP/backport-batch" ./cmd/backport-batch' ] + [ "$(yq -r "$JOB.steps[] | select(.name == \"Fetch the batch head as data\") | .run" "$WF")" = 'git fetch --no-tags --quiet origin "$HEAD_SHA"' ] +} + +@test "the tool is built before the head is fetched, and verify gets it through env" { + names="$(yq -r "$JOB.steps[].name" "$WF")" + [ "$(grep -n 'Build backport-batch from main' <<<"$names" | cut -d: -f1)" -lt "$(grep -n 'Fetch the batch head as data' <<<"$names" | cut -d: -f1)" ] + run_line="$(yq -r "$JOB.steps[] | select(.name == \"Verify every original on the batch\") | .run" "$WF")" + [ "$run_line" = '"$RUNNER_TEMP/backport-batch" verify --line "$BASE_REF" --head "$HEAD_SHA"' ] +} + +@test "no expression reaches a shell in the job" { + runs="$(yq -r "$JOB.steps[] | select(has(\"run\")) | .run" "$WF")" + [ -n "$runs" ] + if grep -q '\${{' <<<"$runs"; then + echo "a run: line carries an expression:" + echo "$runs" + false + fi +} diff --git a/hack/backupstrategy-crd-gate.bats b/hack/backupstrategy-crd-gate.bats index b4ad482a0..fac48f35e 100644 --- a/hack/backupstrategy-crd-gate.bats +++ b/hack/backupstrategy-crd-gate.bats @@ -133,3 +133,39 @@ $tpl: $obj does not gate on crdEstablished for $crd" rm -rf "$tmp" } + +# The controller's DefaultObjectsGate tells an object rendered with a stale S3 +# endpoint (publishing.host changed after the render) by the +# backups.cozystack.io/endpoint annotation, because the endpoint itself sits at +# a different path in every kind. An object the chart renders without it +# counts as stale on every check, so a new template that forgets it would +# force this release every MinForceInterval forever. +@test "every rendered Strategy CR and the Velero BSL carry the rendered endpoint annotation" { + tmp=$(mktemp -d) + + helm template t "$CHART" -n cozy-backup-controller \ + --set backupStorage.bucketNameOverride=ext-bucket \ + --set backupStorage.endpoint=https://s3.example.org \ + --output-dir "$tmp/open" >/dev/null + + objs=$(find "$tmp/open" -name '*.yaml' | sort | while read -r f; do + yq eval-all "select((.apiVersion // \"\" | test(\"^${STRATEGY_API}\")) or .kind == \"BackupStorageLocation\") | .kind + \"/\" + .metadata.name + \" \" + (.metadata.annotations[\"backups.cozystack.io/endpoint\"] // \"\")" "$f" \ + | sed "s#^#$(basename "$f") #" + done) + if ! echo "$objs" | grep -q " BackupStorageLocation/"; then + echo "render produced no BSL; nothing was checked for it" >&2 + false + fi + if ! echo "$objs" | grep -q " CNPG/"; then + echo "render produced no Strategy CR; nothing was checked" >&2 + false + fi + bad=$(echo "$objs" | awk '$3 != "https://s3.example.org"') + if [ -n "$bad" ]; then + echo "objects without the rendered endpoint in backups.cozystack.io/endpoint:" >&2 + echo "$bad" >&2 + false + fi + + rm -rf "$tmp" +} diff --git a/hack/check-commit-trailers.bats b/hack/check-commit-trailers.bats index 4818c0981..85603b8d8 100644 --- a/hack/check-commit-trailers.bats +++ b/hack/check-commit-trailers.bats @@ -520,3 +520,17 @@ EOF fi cleanup } + +# The workflow skips this check on backport branches, whose commits are picked +# verbatim. A fork can name a branch backport-batch/anything, so the batch's +# exemption holds only for a branch of this repository. +@test "the workflow exempts a batch branch only from this repository" { + cond="$(yq -r '.jobs.commit-trailers.if' "$REPO_ROOT/.github/workflows/pre-commit.yml")" + case "$cond" in + *"!(startsWith(github.head_ref, 'backport-batch/') && github.event.pull_request.head.repo.full_name == github.repository)"*) ;; + *) + echo "the batch exemption is not bound to this repository: $cond" >&2 + exit 1 + ;; + esac +} diff --git a/hack/ci-tool-installs-contract.bats b/hack/ci-tool-installs-contract.bats index 009911249..739f72394 100755 --- a/hack/ci-tool-installs-contract.bats +++ b/hack/ci-tool-installs-contract.bats @@ -111,8 +111,9 @@ one_value_of() { @test "every use of the cosign installer is SHA-pinned at one cosign release" { # Without cosign-release the action installs its own default, which moves # with a digest bump of the action. The release jobs verify their signature - # with the cosign docs/release.md asks users for, so the version is named in - # each step, and all steps agree on it and on the action's commit. + # with the cosign docs/release/signatures-and-sbom.md asks users for, so the + # version is named in each step, and all steps agree on it and on the + # action's commit. uses="$(workflow_code 'uses: sigstore/cosign-installer@')" [ -n "$uses" ] bad="$(printf '%s\n' "$uses" | grep --invert-match --extended-regexp 'installer@[0-9a-f]{40} # v[0-9.]+$' || true)" diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats new file mode 100644 index 000000000..e912be42f --- /dev/null +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -0,0 +1,263 @@ +#!/usr/bin/env bats +# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile +# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a +# restore that fails without saying so leaves the rewrite in a cluster-wide +# object and still reports a green run; the next run then refuses to start. +# These run the restore against a stubbed kubectl. + +load test_helper + +LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh +HELPERS=examples/backups/clickhouse/00-helpers.sh + +setup() { + strict_setup + STUB=$(mktemp -d) + # A Corefile carrying the Test's marked block, as the run step leaves it. + jq -n '{apiVersion: "v1", kind: "ConfigMap", + metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"}, + data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \ + >"$STUB/coredns.json" + # The secret lookup follows kubectl's contract: a name together with a + # selector is refused, and a selector-only lookup prints the names in + # OWNED_SECRETS, the Secrets carrying the owner label. + cat >"$STUB/kubectl" <<'EOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;; + replace*) + cp "${@: -1}" "$STUB/replaced.json" + if [ -n "${REPLACE_FAILS:-}" ]; then + echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2 + exit 1 + fi + ;; + *"get secret"*) + # Positionals are "get", "secret" and then any names. + words=0 selector= + while [ $# -gt 0 ]; do + case "$1" in + -l|--selector) selector=$2; shift 2 ;; + -n|--namespace|-o|--output) shift 2 ;; + -*) shift ;; + *) words=$((words + 1)); shift ;; + esac + done + if [ -n "${LOOKUP_FAILS:-}" ]; then + echo 'Unable to connect to the server: dial tcp: i/o timeout' >&2 + exit 1 + fi + if [ -n "$selector" ] && [ "$words" -gt 2 ]; then + echo 'error: name cannot be provided when a selector is specified' >&2 + exit 1 + fi + [ "$selector" = cozystack.io/e2e-owner=clickhouse-system-bucket ] || exit 0 + for s in ${OWNED_SECRETS:-}; do echo "secret/$s"; done + ;; + *"delete secret "*) + if [ -n "${DELETE_FAILS:-}" ]; then + echo 'Error from server (Forbidden): secrets "seaweedfs-s3-ingress-tls" is forbidden' >&2 + exit 1 + fi + ;; + *) echo "unexpected kubectl $*" >&2; exit 2 ;; +esac +EOF + chmod +x "$STUB/kubectl" + export STUB +} + +restore() { + PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c ' + set -euo pipefail + source "$1" + source "$2" + set +e + restore_shared_state + ' _ "$HELPERS" "$LIB" +} + +@test "a CoreDNS restore the API server refuses fails the restore" { + export REPLACE_FAILS=1 + err=$(mktemp) + if restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl replace was refused" >&2 + cat "$err" >&2 + false + fi + if grep -q 'Removed the s3.example.org rewrite' "$err"; then + echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2 + cat "$err" >&2 + false + fi +} + +@test "a CoreDNS restore sends the Corefile back without the marked block" { + restore + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || { + echo "restored Corefile:" >&2 + jq -r '.data.Corefile' "$STUB/replaced.json" >&2 + false + } + # The resourceVersion read travels with the replace, so a concurrent + # writer makes it fail rather than be overwritten. + [ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ] +} + +@test "the restore deletes the Ingress TLS Secret the Test created" { + OWNED_SECRETS=seaweedfs-s3-ingress-tls restore + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state did not delete the Secret carrying the owner label; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + } +} + +@test "the restore leaves an Ingress TLS Secret without the owner label alone" { + for owned in '' e2e-other seaweedfs-s3-ingress-tls-old; do + : >"$STUB/calls" + OWNED_SECRETS=$owned restore + if grep -q 'delete secret' "$STUB/calls"; then + echo "restore_shared_state deleted a Secret although the labelled ones are '$owned'; kubectl calls:" >&2 + cat "$STUB/calls" >&2 + false + fi + done +} + +@test "a Secret delete the API server refuses fails the restore" { + err=$(mktemp) + if OWNED_SECRETS=seaweedfs-s3-ingress-tls DELETE_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although kubectl delete secret was refused" >&2 + cat "$err" >&2 + false + fi + grep -qE '^-n tenant-root delete secret seaweedfs-s3-ingress-tls( |$)' "$STUB/calls" || { + echo "restore_shared_state failed before it reached the Secret delete; kubectl calls:" >&2 + cat "$STUB/calls" "$err" >&2 + false + } +} + +@test "a Secret lookup that fails fails the restore" { + err=$(mktemp) + if LOOKUP_FAILS=1 restore 2>"$err"; then + echo "restore_shared_state returned 0 although it could not tell whether the Secret is left over" >&2 + cat "$err" >&2 + false + fi + grep -q 'could not list the Secrets labelled' "$err" || { + echo "restore_shared_state failed without naming the Secret lookup:" >&2 + cat "$err" >&2 + false + } +} + +@test "the run step's finally restores the shared objects" { + # When the step ceiling fires, Chainsaw kills the shell and the EXIT trap + # of system-bucket-roundtrip.sh never runs; the step's finally still does. + # Chainsaw runs a script's content with sh from the suite's directory. + suite=hack/e2e-chainsaw/clickhouse + finally=$(yq 'select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") + | .spec.steps[] | select(.name == "run") | .finally[] + | select(.description == "restore the host CoreDNS and the S3 Ingress TLS Secret") | .script.content' \ + "$suite/chainsaw-test.yaml") + [ -n "$finally" ] || { echo "the run step has no finally restoring the shared objects" >&2; false; } + (cd "$suite" && PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$finally") + [ -f "$STUB/replaced.json" ] || { + echo "the run step's finally did not put the Corefile back" >&2 + false + } + jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null +} + +# The Altinity driver names the Backup after the BackupJob, and a Backup left +# behind under that name records an earlier run's archive, which fails the next +# run's BackupJob. It is reclaimed by name, whether or not the BackupJob +# recorded it in status.backupRef. +@test "the round-trip reclaims its Backup by name before and after the run" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + # Nothing is left over from an earlier run, and no BackupJob recorded a + # backupRef. + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + for selector in \ + 'select(.name == "preconditions") | .try[].script.content' \ + 'select(.name == "run") | .finally[] | select(.description == "system-bucket round-trip cleanup") | .script.content'; do + : >"$calls" + script=$(yq "$test | $selector" "$suite") + [ -n "$script" ] || { echo "no script at: $selector" >&2; false; } + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + grep -qE '^-n tenant-root delete backup sysbucket-backup( |$)' "$calls" || { + echo "the script at '$selector' does not delete Backup sysbucket-backup by name; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + done +} + +# cert-manager issues the Secret of a Certificate that still exists again as +# soon as it is deleted. Chainsaw deletes the Certificate it applied, and waits +# for it, before it runs the cleanup ops of the same step, so the Secret is +# deleted there and nowhere earlier. +@test "the Certificate's Secret is deleted after Chainsaw deletes the Certificate" { + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + applies=$(yq "$test | select(.name == \"create-certificate-and-probe\") | .try[] | select(has(\"apply\")) | .apply.file" "$suite") + [ "$applies" = system-bucket-tls.yaml ] || { + echo "step create-certificate-and-probe no longer applies system-bucket-tls.yaml alone: $applies" >&2 + false + } + deletes=$(yq -o=json -I=0 "$test | select(.name == \"create-certificate-and-probe\") | .cleanup" "$suite") + [ "$deletes" = '[{"delete":{"ref":{"apiVersion":"v1","kind":"Secret","name":"e2e-system-bucket-s3-tls"}}}]' ] || { + echo "the cleanup of step create-certificate-and-probe is not the one delete of Secret e2e-system-bucket-s3-tls: $deletes" >&2 + false + } + # The preconditions run before the apply, when Chainsaw holds no Certificate. + if yq "$test | select(.name != \"preconditions\") | (.try[], .finally[]?) | .script.content // \"\"" "$suite" | grep -qE 'delete +(secret +e2e-system-bucket-s3-tls|certificate +e2e-system-bucket-s3)( |$)'; then + echo "a step script deletes the Certificate or its Secret while Chainsaw still holds the Certificate" >&2 + false + fi +} + +# Chainsaw registers an applied object for deletion only when its apply created +# it; an object left by a run killed past its cleanup is patched and kept. The +# Certificate would then outlive every later run, re-issuing its Secret after +# each step cleanup, and a probe Pod whose `sleep` had ended never turns Ready +# again, so the preconditions remove all three before the apply. +@test "the preconditions remove the TLS objects and probe Pod a killed run left" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + : >"$calls" + script=$(yq "$test | select(.name == \"preconditions\") | .try[].script.content" "$suite") + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + line_of() { grep -nE "^-n tenant-root delete $1 --ignore-not-found --wait=true( |$)" "$calls" | head -n1 | cut -d: -f1; } + cert=$(line_of 'certificate e2e-system-bucket-s3') + secret=$(line_of 'secret e2e-system-bucket-s3-tls') + pod=$(line_of 'pod e2e-system-bucket-probe') + [ -n "$cert" ] && [ -n "$secret" ] && [ -n "$pod" ] || { + echo "the preconditions do not delete, and wait for, the Certificate, its Secret and the probe Pod; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + [ "$cert" -lt "$secret" ] || { + echo "the preconditions delete Secret e2e-system-bucket-s3-tls before its Certificate, which cert-manager then issues again" >&2 + false + } +} diff --git a/hack/close-manual-backport.bats b/hack/close-manual-backport.bats new file mode 100644 index 000000000..93127ebd6 --- /dev/null +++ b/hack/close-manual-backport.bats @@ -0,0 +1,367 @@ +#!/usr/bin/env bats +# Tests for hack/close-manual-backport.sh, which closes a pull request opened by +# hand against a release branch by someone outside the team. +# +# gh is stubbed: every call is recorded, the permission endpoint answers with +# $PERM_JSON (or fails under PERM_FAIL), and any other `gh api` with $PR_JSON. +# A case that expects the pull request to stay open asserts exactly which reads +# gh recorded, so a close that slips past a guard cannot go unseen. +# +# Run with: bats hack/close-manual-backport.bats + +load test_helper + +SCRIPT="${BATS_TEST_DIRNAME}/close-manual-backport.sh" +WF="${BATS_TEST_DIRNAME}/../.github/workflows/pr-labeler.yaml" +POLICY_URL="https://github.com/cozystack/cozystack/blob/main/docs/release/backports.md" +PERM_CALL="api repos/cozystack/cozystack/collaborators/someone/permission" + +# The close-manual-backport job of pr-labeler.yaml, comment lines dropped so a +# setting demoted to a comment cannot satisfy an assertion. +guard_job() { + awk '/^ close-manual-backport:$/ { f = 1; next } f && /^ [^ ]/ { exit } f' "$WF" \ + | grep -v '^[[:space:]]*#' +} + +_stub_gh() { + mkdir -p "$1" + cat > "$1/gh" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >> "$GH_CALLS" +case "$1 $2" in + "api repos/"*/collaborators/*/permission) + if [ -n "${PERM_FAIL:-}" ]; then + echo 'gh: Not Found (HTTP 404)' >&2 + exit 1 + fi + cat "$PERM_JSON" + ;; + "api "*) cat "$PR_JSON" ;; +esac +EOF + chmod +x "$1/gh" +} + +setup() { + strict_setup + _stub_gh "${BATS_TEST_TMPDIR}/bin" + export PATH="${BATS_TEST_TMPDIR}/bin:$PATH" + export GH_CALLS="${BATS_TEST_TMPDIR}/calls" + export PR_JSON="${BATS_TEST_TMPDIR}/pr.json" + export PERM_JSON="${BATS_TEST_TMPDIR}/perm.json" + unset PERM_FAIL + set_permission read read + EVENT="${BATS_TEST_TMPDIR}/event.json" + # A run inside GitHub Actions has its own event here; never read it. + export GITHUB_EVENT_PATH="$EVENT" + : > "$GH_CALLS" +} + +# The permission endpoint's answer: legacy permission, then role_name. +set_permission() { + printf '{"permission":"%s","role_name":"%s","user":{"login":"someone"}}\n' "$1" "$2" > "$PERM_JSON" +} + +# Writes a pull_request_target event to $EVENT and its pull request object to +# $PR_JSON. Arguments: base ref, author association, author type, labels as a +# JSON array of names, and the author's login, someone by default. "-" leaves +# the association or the type out. +make_event() { + jq -n --arg base "$1" --arg assoc "$2" --arg type "$3" --argjson labels "$4" --arg login "${5:-someone}" ' + ({number: 42, state: "open", + base: {ref: $base, repo: {full_name: "cozystack/cozystack"}}, + user: {login: $login, type: $type}, + author_association: $assoc, + labels: ($labels | map({name: .}))} + | if $assoc == "-" then del(.author_association) else . end + | if $type == "-" then del(.user.type) else . end) as $pr + | {action: "opened", number: 42, pull_request: $pr, + repository: {full_name: "cozystack/cozystack"}}' > "$EVENT" + jq '.pull_request' "$EVENT" > "$PR_JSON" +} + +assert_left_open() { + [ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; } + if [ -s "$GH_CALLS" ]; then + echo "FAIL: gh was called:" + cat "$GH_CALLS" + false + fi + case "$output" in + *"leaving it open"*) ;; + *) echo "FAIL: no reason given: $output"; false ;; + esac +} + +# Failed after the permission read, which is then the only gh call, with +# nothing closed and why said. +assert_failed_after_permission() { + [ "$status" -eq 1 ] || { echo "FAIL: exit $status: $output"; false; } + [ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; } + case "$output" in + *"closing nothing"*) ;; + *) echo "FAIL: no reason given: $output"; false ;; + esac +} + +# Left open after the permission read, which is then the only gh call. +assert_left_open_after_permission() { + [ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; } + [ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; } + case "$output" in + *"leaving it open"*) ;; + *) echo "FAIL: no reason given: $output"; false ;; + esac +} + +@test "closes a pull request to release-1.6 from outside the team, linking the policy" { + for assoc in CONTRIBUTOR FIRST_TIME_CONTRIBUTOR FIRST_TIMER NONE; do + : > "$GH_CALLS" + make_event release-1.6 "$assoc" User '["kind/bug"]' + run "$SCRIPT" "$EVENT" + [ "$status" -eq 0 ] || { echo "FAIL: $assoc: exit $status: $output"; false; } + [ "$(wc -l < "$GH_CALLS" | tr -d ' ')" -eq 2 ] || { echo "FAIL: $assoc:"; cat "$GH_CALLS"; false; } + [ "$(sed -n 1p "$GH_CALLS")" = "$PERM_CALL" ] + sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment Thank you' + grep -qF "$POLICY_URL" "$GH_CALLS" + grep -qF 'a maintainer can reopen it.' "$GH_CALLS" + if grep -qF '—' "$GH_CALLS"; then echo "FAIL: the comment carries an em dash"; false; fi + done +} + +@test "leaves OWNER, MEMBER and COLLABORATOR pull requests open" { + for assoc in OWNER MEMBER COLLABORATOR; do + : > "$GH_CALLS" + make_event release-1.6 "$assoc" User '[]' + run "$SCRIPT" "$EVENT" + assert_left_open + done +} + +@test "leaves a bot's pull request open" { + make_event release-1.6 NONE Bot '[]' + run "$SCRIPT" "$EVENT" + assert_left_open +} + +# The scheduled batch opens its pull requests as the cozystack-ci App, which +# GitHub can report as a contributor once it has commits here. +@test "leaves the batch's own pull request open" { + for assoc in NONE CONTRIBUTOR; do + : > "$GH_CALLS" + make_event release-1.7 "$assoc" Bot '[]' 'cozystack-ci[bot]' + run "$SCRIPT" "$EVENT" + assert_left_open + done +} + +# No label keeps a pull request open: what keeps one is a maintainer +# reopening it, which the guard does not act on. +@test "closes a pull request whatever labels it carries" { + make_event release-1.6 CONTRIBUTOR User '["kind/bug","release-only","backport/include"]' + run "$SCRIPT" "$EVENT" + [ "$status" -eq 0 ] || { echo "FAIL: exit $status: $output"; false; } + sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment ' \ + || { echo "FAIL: not closed:"; cat "$GH_CALLS"; false; } +} + +@test "acts only on an opened event, so a maintainer keeps a fix by reopening it" { + for action in reopened synchronize edited labeled ''; do + : > "$GH_CALLS" + make_event release-1.6 CONTRIBUTOR User '[]' + jq --arg a "$action" 'if $a == "" then del(.action) else .action = $a end' "$EVENT" > "$EVENT.tmp" + mv "$EVENT.tmp" "$EVENT" + run "$SCRIPT" "$EVENT" + assert_left_open + case "$output" in + *"not opened"*) ;; + *) echo "FAIL: reason does not name the event ($action): $output"; false ;; + esac + done +} + +@test "leaves pull requests to main and to other branches open" { + for base in main feature-release-1.6 releases; do + : > "$GH_CALLS" + make_event "$base" CONTRIBUTOR User '[]' + run "$SCRIPT" "$EVENT" + assert_left_open + done +} + +@test "does nothing when the author association is missing" { + make_event release-1.6 - User '[]' + run "$SCRIPT" "$EVENT" + assert_left_open + case "$output" in + *"no author association"*) ;; + *) echo "FAIL: reason does not name the association: $output"; false ;; + esac +} + +@test "does nothing on an unknown association, a missing author type or an odd login" { + make_event release-1.6 SOMETHING_NEW User '[]' + run "$SCRIPT" "$EVENT" + assert_left_open + + : > "$GH_CALLS" + make_event release-1.6 CONTRIBUTOR - '[]' + run "$SCRIPT" "$EVENT" + assert_left_open + + # The login goes into the API path, so only a GitHub login shape is asked about. + : > "$GH_CALLS" + make_event release-1.6 CONTRIBUTOR User '[]' + jq '.pull_request.user.login = "someone/../../orgs/x"' "$EVENT" > "$EVENT.tmp" + mv "$EVENT.tmp" "$EVENT" + run "$SCRIPT" "$EVENT" + assert_left_open +} + +# author_association shows an organization member whose membership is private +# as a CONTRIBUTOR. `permission` alone would also pass a triage member, which +# GitHub folds into read. +@test "a private member reported as CONTRIBUTOR stays open when their access is above read" { + for answer in "write write" "admin admin" "write maintain" "read triage" "read some-custom-role"; do + : > "$GH_CALLS" + set -- $answer + set_permission "$1" "$2" + make_event release-1.6 CONTRIBUTOR User '[]' + run "$SCRIPT" "$EVENT" + assert_left_open_after_permission + case "$output" in + *"permission on cozystack/cozystack is $1 (role $2)"*) ;; + *) echo "FAIL: reason does not name the access ($answer): $output"; false ;; + esac + done +} + +@test "a failed or unreadable permission answer is an error and closes nothing" { + make_event release-1.6 CONTRIBUTOR User '[]' + + export PERM_FAIL=1 + run "$SCRIPT" "$EVENT" + assert_failed_after_permission + case "$output" in + *"cannot read someone's permission"*) ;; + *) echo "FAIL: reason does not name the failed read: $output"; false ;; + esac + unset PERM_FAIL + + for answer in 'not json' '{}' '{"permission":"read"}' '{"role_name":"read"}' '{"permission":"read","role_name":null}' '[]'; do + : > "$GH_CALLS" + printf '%s\n' "$answer" > "$PERM_JSON" + run "$SCRIPT" "$EVENT" + assert_failed_after_permission + case "$output" in + *"does not give both permission and role_name"*) ;; + *) echo "FAIL: reason does not name the unreadable answer ($answer): $output"; false ;; + esac + done +} + +@test "an author with read or no access is closed" { + for answer in "read read" "none none" "read none" "none read"; do + : > "$GH_CALLS" + set -- $answer + set_permission "$1" "$2" + make_event release-1.6 CONTRIBUTOR User '[]' + run "$SCRIPT" "$EVENT" + [ "$status" -eq 0 ] || { echo "FAIL: $answer: exit $status: $output"; false; } + [ "$(sed -n 1p "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: $answer:"; cat "$GH_CALLS"; false; } + sed -n 2p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment ' \ + || { echo "FAIL: $answer: not closed:"; cat "$GH_CALLS"; false; } + done +} + +@test "--dry-run never calls a write verb" { + make_event release-1.6 CONTRIBUTOR User '[]' + + # From the event the one gh call allowed is the permission read. + run "$SCRIPT" --dry-run "$EVENT" + [ "$status" -eq 0 ] + [ "$(cat "$GH_CALLS")" = "$PERM_CALL" ] || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; } + case "$output" in + *"would close PR #42 in cozystack/cozystack"*"$POLICY_URL"*) ;; + *) echo "FAIL: dry run did not report the close: $output"; false ;; + esac + + # By number, the pull request read and the permission read. + : > "$GH_CALLS" + run "$SCRIPT" --dry-run --pr 42 + [ "$status" -eq 0 ] + [ "$(cat "$GH_CALLS")" = "$(printf 'api repos/{owner}/{repo}/pulls/42\n%s' "$PERM_CALL")" ] \ + || { echo "FAIL: gh calls:"; cat "$GH_CALLS"; false; } + case "$output" in + *"would close PR #42"*) ;; + *) echo "FAIL: dry run did not report the close: $output"; false ;; + esac +} + +@test "--pr reads the pull request with gh and closes it" { + make_event release-1.6 CONTRIBUTOR User '[]' + run "$SCRIPT" --pr 42 + [ "$status" -eq 0 ] + [ "$(wc -l < "$GH_CALLS" | tr -d ' ')" -eq 3 ] + [ "$(sed -n 1p "$GH_CALLS")" = 'api repos/{owner}/{repo}/pulls/42' ] + [ "$(sed -n 2p "$GH_CALLS")" = "$PERM_CALL" ] + sed -n 3p "$GH_CALLS" | grep -q '^pr close 42 --repo cozystack/cozystack --comment ' +} + +@test "leaves a pull request that is already closed alone" { + make_event release-1.6 CONTRIBUTOR User '[]' + jq '.state = "closed"' "$PR_JSON" > "$PR_JSON.tmp" + mv "$PR_JSON.tmp" "$PR_JSON" + run "$SCRIPT" --pr 42 + [ "$status" -eq 0 ] + [ "$(cat "$GH_CALLS")" = 'api repos/{owner}/{repo}/pulls/42' ] + case "$output" in + *"is not open"*) ;; + *) echo "FAIL: $output"; false ;; + esac +} + +@test "an unreadable event is an error and closes nothing" { + printf 'not json' > "$EVENT" + run "$SCRIPT" "$EVENT" + [ "$status" -eq 1 ] + if [ -s "$GH_CALLS" ]; then echo "FAIL: gh was called:"; cat "$GH_CALLS"; false; fi +} + +@test "a non-numeric --pr is a usage error" { + run "$SCRIPT" --pr '42; gh pr close 1' + [ "$status" -eq 2 ] + if [ -s "$GH_CALLS" ]; then echo "FAIL: gh was called:"; cat "$GH_CALLS"; false; fi +} + +@test "pr-labeler runs the guard only on opened pull requests to release-*" { + job="$(guard_job)" + [ -n "$job" ] + printf '%s\n' "$job" | grep -qxF " github.event.action == 'opened' &&" + if printf '%s\n' "$job" | grep -qF 'reopened'; then echo "FAIL: the job runs on reopened"; false; fi + printf '%s\n' "$job" | grep -qF "startsWith(github.event.pull_request.base.ref, 'release-')" + printf '%s\n' "$job" | grep -qxF ' run: hack/close-manual-backport.sh "$GITHUB_EVENT_PATH"' +} + +# pull_request_target hands a write token to a fork's pull request. The job is +# safe only while it holds no more than it needs and never reaches the head. +@test "the guard job holds the least token and never checks out the pull request" { + job="$(guard_job)" + [ -n "$job" ] + + perms="$(printf '%s\n' "$job" | awk '/^ permissions:$/ { f = 1; next } f && /^ [^ ]/ { exit } f' \ + | sed 's/[[:space:]]*#.*$//' | sort)" + [ "$perms" = "$(printf ' contents: read\n pull-requests: write')" ] || { + echo "FAIL: permissions are:"; printf '%s\n' "$perms"; false + } + + # actions/checkout without a ref takes the event's own, which on + # pull_request_target is the base repository; a ref is the way to the head. + if printf '%s\n' "$job" | grep -qE '^[[:space:]]+ref:'; then echo "FAIL: the checkout names a ref"; false; fi + printf '%s\n' "$job" | grep -qxF ' persist-credentials: false' + + # The token is the only expression: event fields reach the script through + # the JSON file, never through the shell. + [ "$(printf '%s\n' "$job" | grep -oF '${{' | wc -l | tr -d ' ')" -eq 1 ] + printf '%s\n' "$job" | grep -qxF ' GH_TOKEN: ${{ github.token }}' +} diff --git a/hack/close-manual-backport.sh b/hack/close-manual-backport.sh new file mode 100755 index 000000000..3b1770562 --- /dev/null +++ b/hack/close-manual-backport.sh @@ -0,0 +1,156 @@ +#!/bin/sh +# Close a pull request opened by hand against a release branch by someone +# outside the team. Fixes reach the release lines through the backport batch +# (docs/release/backports.md), so a hand-made backport is work nobody needs. It +# acts only on an `opened` event: a maintainer keeps a fix that exists only on +# a release line by reopening it. +# +# Usage: close-manual-backport.sh [--dry-run] [EVENT_JSON] +# close-manual-backport.sh [--dry-run] --pr NUMBER +# +# EVENT_JSON defaults to $GITHUB_EVENT_PATH and is a pull_request or +# pull_request_target payload. --pr reads the same REST pull request object +# with `gh api`, because `gh pr view --json` has no author association. Only +# JSON is read, that object and the author's repository permission: nothing +# from the pull request is checked out or run. +# +# The pull request is closed only when the event, if there is one, is +# `opened`, the pull request is open and targets release-*, its author's +# association is not OWNER, MEMBER or COLLABORATOR, its author is not a bot, +# and GitHub answers that the author's permission and role on the repository +# are both read or none. A missing field or a value this script does not know +# leaves it open and exits 0. Exits 1, closing nothing, when the input or +# GitHub's answer on the author cannot be read, 2 on a usage error, and with +# gh's status when the close fails. +set -eu + +usage() { + echo "usage: close-manual-backport.sh [--dry-run] [EVENT_JSON | --pr NUMBER]" >&2 + exit 2 +} + +leave() { + echo "close-manual-backport: $*; leaving it open." + exit 0 +} + +dry_run=false +pr_number= +event= +while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) dry_run=true ;; + --pr) + [ "$#" -ge 2 ] || usage + pr_number=$2 + shift + ;; + -*) usage ;; + *) + [ -z "$event" ] || usage + event=$1 + ;; + esac + shift +done + +command -v jq >/dev/null 2>&1 || { echo "close-manual-backport: jq is required" >&2; exit 1; } + +if [ -n "$pr_number" ]; then + [ -z "$event" ] || usage + case "$pr_number" in '' | *[!0-9]*) usage ;; esac + if ! pr=$(gh api "repos/{owner}/{repo}/pulls/$pr_number"); then + echo "close-manual-backport: cannot read PR #$pr_number with gh" >&2 + exit 1 + fi +else + event=${event:-${GITHUB_EVENT_PATH:-}} + [ -n "$event" ] || usage + if ! pr=$(jq -c '.pull_request' "$event" 2>/dev/null); then + echo "close-manual-backport: cannot read the event JSON at $event" >&2 + exit 1 + fi + action=$(jq -r '.action | strings' "$event" 2>/dev/null || true) + [ "$action" = opened ] || leave "the event is ${action:-missing}, not opened" +fi + +# A filter that fails on an unexpected shape prints nothing, which every check +# below reads as a missing field. +json_field() { + printf '%s' "$1" | jq -r "$2" 2>/dev/null || true +} + +field() { + json_field "$pr" "$1" +} + +[ "$(field 'type')" = object ] || leave "the input carries no pull request" +number=$(field '.number | select(type == "number" and . > 0 and . == floor) | tostring') +[ -n "$number" ] || leave "the pull request carries no number" + +base=$(field '.base.ref | strings') +case "$base" in + release-*) ;; + '') leave "PR #$number carries no base ref" ;; + *) leave "PR #$number targets $base, not a release branch" ;; +esac + +state=$(field '.state | strings') +[ "$state" = open ] || leave "PR #$number is not open (state: ${state:-missing})" + +assoc=$(field '.author_association | strings') +case "$assoc" in + OWNER | MEMBER | COLLABORATOR) leave "PR #$number is from a repository $assoc" ;; + CONTRIBUTOR | FIRST_TIME_CONTRIBUTOR | FIRST_TIMER | NONE) ;; + '') leave "PR #$number carries no author association" ;; + *) leave "PR #$number has an author association this script does not know ($assoc)" ;; +esac + +user_type=$(field '.user.type | strings') +case "$user_type" in + User) ;; + Bot) leave "PR #$number is from a bot" ;; + '') leave "PR #$number carries no author type" ;; + *) leave "PR #$number has an author type this script does not know ($user_type)" ;; +esac + +repo=$(field '.base.repo.full_name | strings') +case "$repo" in + */*/* | *[!A-Za-z0-9._/-]* | /* | */) leave "PR #$number names no usable base repository" ;; + */*) ;; + *) leave "PR #$number names no usable base repository" ;; +esac +login=$(field '.user.login | strings') +case "$login" in + '' | *[!A-Za-z0-9-]*) leave "PR #$number carries no usable author login" ;; +esac + +# author_association reports an organization member whose membership is +# private as a CONTRIBUTOR, so the author's own access to the repository is the +# check that decides. `permission` folds maintain into write and triage into +# read, which is why `role_name` has to agree. An answer that cannot be read +# fails the job: left open in silence, the pull request would look decided. +if ! access=$(gh api "repos/$repo/collaborators/$login/permission"); then + echo "close-manual-backport: PR #$number: cannot read $login's permission on $repo; closing nothing" >&2 + exit 1 +fi +permission=$(json_field "$access" '.permission | strings') +role=$(json_field "$access" '.role_name | strings') +if [ -z "$permission" ] || [ -z "$role" ]; then + echo "close-manual-backport: PR #$number: GitHub's answer on $login's permission on $repo does not give both permission and role_name; closing nothing" >&2 + exit 1 +fi +case "$permission:$role" in + read:read | read:none | none:read | none:none) ;; + *) leave "PR #$number is from $login, whose permission on $repo is $permission (role $role)" ;; +esac + +policy="https://github.com/$repo/blob/main/docs/release/backports.md" +comment="Thank you for the contribution! Backports are automatic: fixes merged to \`main\` reach the release lines the batch runs for on their own, as the [backport policy]($policy) describes, so this pull request is closed. If the fix only makes sense on a release line and has no counterpart on \`main\`, a maintainer can reopen it." + +if [ "$dry_run" = true ]; then + printf 'close-manual-backport: would close PR #%s in %s with this comment:\n%s\n' "$number" "$repo" "$comment" + exit 0 +fi +gh pr close "$number" --repo "$repo" --comment "$comment" +echo "close-manual-backport: closed PR #$number in $repo." diff --git a/hack/cni-plugins-staging-contract.bats b/hack/cni-plugins-staging-contract.bats index 2f3acb2f1..ce63e45d2 100644 --- a/hack/cni-plugins-staging-contract.bats +++ b/hack/cni-plugins-staging-contract.bats @@ -22,7 +22,10 @@ extract_has() { } DOCKERFILE="$PWD/packages/system/multus/images/multus-cni/Dockerfile" -KUBEOVN_VALUES="$PWD/packages/system/kubeovn/values.yaml" +# The chart's appVersion, not the values.yaml image tag: a release build +# restamps that tag to the Cozystack version (v1.7.0), the chart keeps the +# kube-ovn release it vendors. +KUBEOVN_CHART="$PWD/packages/system/kubeovn/charts/kube-ovn/Chart.yaml" DOC="$PWD/packages/system/multus/README.md" PLATFORM_VALUES="$PWD/packages/core/platform/values.yaml" TEMPLATE="$PWD/packages/system/multus/templates/multus-daemonset-thick.yml" @@ -347,13 +350,12 @@ PATCHFILE="$PWD/packages/system/multus/patches/customize-deployment.patch" checked=$(sed -n 's|^# kube-ovn-cni-plugins-checked-against: *\([^ ]*\) *$|\1|p' "$DOCKERFILE") [ -n "$checked" ] - vendored=$(yq -r '.global.images.kubeovn.tag' "$KUBEOVN_VALUES" | sed 's|@.*||') - [ -n "$vendored" ] - [ "$vendored" != "null" ] + vendored=$(awk '$1 == "appVersion:" {gsub(/"/, "", $2); print "v" $2}' "$KUBEOVN_CHART") + [ "$vendored" != "v" ] if [ "$checked" != "$vendored" ]; then echo "the multus CNI plugin pin was checked against kube-ovn $checked," >&2 - echo "but packages/system/kubeovn/values.yaml now vendors $vendored." >&2 + echo "but packages/system/kubeovn/charts/kube-ovn/Chart.yaml now vendors $vendored." >&2 echo >&2 echo "Read CNI_PLUGINS_VERSION in kube-ovn $vendored's base image" >&2 echo "(dist/images/Dockerfile.base). If it still matches the" >&2 diff --git a/hack/console-nginx-header-buffers.bats b/hack/console-nginx-header-buffers.bats index f8ce4d949..21932fc25 100644 --- a/hack/console-nginx-header-buffers.bats +++ b/hack/console-nginx-header-buffers.bats @@ -37,11 +37,11 @@ nginx_size_kib() { cd "$REPO_ROOT" conf=$(awk ' - /^COPY <<.EOF. \/etc\/nginx\/conf\.d\/default\.conf$/ { inblock = 1; next } + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } inblock && /^EOF$/ { inblock = 0; next } inblock { print } ' "$CONSOLE_CONTAINERFILE") - [ -n "$conf" ] || { echo "no /etc/nginx/conf.d/default.conf heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; } + [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc found in $CONSOLE_CONTAINERFILE" >&2; exit 1; } # An extraction that silently grabbed the wrong slice would let every # assertion below pass on an empty-ish config, so anchor on a line the diff --git a/hack/console-nginx-observability-proxy.bats b/hack/console-nginx-observability-proxy.bats new file mode 100644 index 000000000..de2348dd7 --- /dev/null +++ b/hack/console-nginx-observability-proxy.bats @@ -0,0 +1,243 @@ +#!/usr/bin/env bats +# The console nginx sends the logs and metrics roots of the observability +# proxy to that proxy instead of the API server. The routing lives in the +# nginx config baked into the console image, which no chart template and no +# helm-unittest case can see, so it is pinned here. +# +# Four properties are checked. The location regex must take the four path +# shapes the proxy serves and leave every ordinary Kubernetes API path to the +# /apis and /api prefix locations; nginx runs the regex locations whenever the +# longest prefix match is a plain prefix, so the prefix locations must not +# carry ^~, which would stop that. The location must stream: tail and watch +# style responses stall behind proxy buffering and die at the default 60s +# read timeout. And the upstream must be a variable resolved per request, +# because nginx resolves a literal upstream host once at startup and refuses +# to start when it does not resolve, which is the case on every cluster that +# does not run the proxy. Finally, when nginx cannot reach the proxy it +# answers with the Kubernetes Status the console reads as not installed, +# while the proxy's own 502 passes through; that check runs the extracted +# locations in a real nginx and is skipped where none is installed. +# +# The regex is evaluated with perl, whose syntax nginx's PCRE accepts for the +# constructs used here. +# Run with: bats hack/console-nginx-observability-proxy.bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile" + +console_nginx_template() { + awk ' + /^COPY <<.EOF. \/etc\/nginx\/templates\/default\.conf\.template$/ { inblock = 1; next } + inblock && /^EOF$/ { inblock = 0; next } + inblock { print } + ' "$REPO_ROOT/$CONSOLE_CONTAINERFILE" +} + +# Prints the regex of every `location ~ {` line, one per line. +regex_locations() { + printf '%s\n' "$1" | awk '$1 == "location" && $2 == "~" && $4 == "{" { print $3 }' +} + +# Prints the body of the location block whose regex is $2. +location_body() { + # Through ENVIRON rather than -v, which would expand the backslashes. + printf '%s\n' "$1" | LOCATION_RE="$2" awk ' + $1 == "location" && $2 == "~" && $3 == ENVIRON["LOCATION_RE"] { inblock = 1; next } + inblock && $1 == "}" { exit } + inblock { print } + ' +} + +matches() { + perl -e 'exit(($ARGV[1] =~ /$ARGV[0]/) ? 0 : 1)' "$1" "$2" +} + +@test "console nginx routes exactly the observability proxy roots to the proxy" { + conf=$(console_nginx_template) + [ -n "$conf" ] || { echo "no /etc/nginx/templates/default.conf.template heredoc in $CONSOLE_CONTAINERFILE" >&2; exit 1; } + + regexes=$(regex_locations "$conf") + [ "$(printf '%s\n' "$regexes" | grep -c .)" -eq 1 ] \ + || { echo "expected exactly one regex location, got: $regexes" >&2; exit 1; } + re=$regexes + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/metrics/api/v1/query_range \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log/select/logsql/tail \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/vminstances/vm1/metrics/api/v1/query + do + matches "$re" "$path" || { echo "$path does not reach the proxy" >&2; exit 1; } + done + + for path in \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs \ + /apis/cozystack.io/v1alpha1/namespaces/tenant-foo/workloadmonitors/db \ + /apis/cozystack.io/v1alpha1/applicationdefinitions/postgres \ + /apis/cozystack.io/v1alpha1/watch/namespaces/tenant-foo/logs/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/log/ \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/status \ + /apis/apps.cozystack.io/v1alpha1/namespaces/tenant-foo/postgreses/db/log \ + /apis/apps.cozystack.io/v1alpha1/watch/namespaces/tenant-foo/postgreses/db/log/ \ + /apis/metrics.k8s.io/v1beta1/namespaces/tenant-foo/pods/ \ + /api/v1/namespaces/tenant-foo/pods/db-0/log \ + /k8s/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query + do + if matches "$re" "$path"; then + echo "FAIL: $path would reach the proxy instead of the API server" >&2 + false + fi + done + + # A ^~ on the /apis or /api prefix would stop nginx from trying the regex. + if printf '%s\n' "$conf" | grep -Eq '^[[:space:]]*location[[:space:]]+\^~'; then + echo "FAIL: a ^~ prefix location disables the regex location" >&2 + false + fi +} + +@test "the observability proxy location streams, forwards the token and resolves per request" { + conf=$(console_nginx_template) + re=$(regex_locations "$conf") + body=$(location_body "$conf" "$re") + [ -n "$body" ] || { echo "no body found for location ~ $re" >&2; exit 1; } + + for directive in \ + 'proxy_http_version 1.1;' \ + 'proxy_buffering off;' \ + 'proxy_read_timeout 86400s;' \ + 'proxy_send_timeout 86400s;' + do + printf '%s\n' "$body" | grep -Fqx " $directive" \ + || { echo "the proxy location lacks '$directive'" >&2; exit 1; } + done + + # nginx forwards client headers unless told otherwise; the proxy needs the + # Authorization header the gatekeeper set. + if printf '%s\n' "$body" | grep -Eq 'proxy_set_header[[:space:]]+Authorization|proxy_pass_request_headers[[:space:]]+off'; then + echo "FAIL: the proxy location drops or rewrites the Authorization header" >&2 + false + fi + + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*resolver[[:space:]]+\$\{NGINX_LOCAL_RESOLVERS\}' \ + || { echo "the proxy location sets no resolver from the pod's resolv.conf" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*set[[:space:]]+\$observability_proxy[[:space:]]+observability-proxy\.cozy-observability-proxy\.svc\.\$\{CLUSTER_DOMAIN\}:80;' \ + || { echo "the proxy location does not name the Service by its fully qualified name" >&2; exit 1; } + printf '%s\n' "$body" | grep -Eq '^[[:space:]]*proxy_pass[[:space:]]+http://\$observability_proxy;' \ + || { echo "the proxy location does not proxy_pass through the variable" >&2; exit 1; } +} + +@test "the console image defines every variable its nginx template substitutes" { + cd "$REPO_ROOT" + conf=$(console_nginx_template) + + # The image's entrypoint substitutes only variables present in the + # environment; one that is not would reach nginx as ${NAME} and fail it. + grep -Eq '^ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image does not ask its entrypoint for NGINX_LOCAL_RESOLVERS" >&2; exit 1; } + grep -Eq '^ENV CLUSTER_DOMAIN=cozy\.local$' "$CONSOLE_CONTAINERFILE" \ + || { echo "the image has no CLUSTER_DOMAIN default" >&2; exit 1; } + + used=$(printf '%s\n' "$conf" | grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*\}' | sort -u) + expected=$(printf '%s\n' '${CLUSTER_DOMAIN}' '${NGINX_LOCAL_RESOLVERS}') + [ "$used" = "$expected" ] \ + || { echo "template substitutes: $used; the image defines: $expected" >&2; exit 1; } +} + +# Prints the named location $2 of the template $1, braces included. +named_location() { + printf '%s\n' "$1" | LOCATION_NAME="$2" awk ' + $1 == "location" && $2 == ENVIRON["LOCATION_NAME"] { inblock = 1 } + inblock { print } + inblock && $1 == "}" { exit } + ' +} + +free_port() { + python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])' +} + +# Starts nginx with the observability locations of the console template, the +# upstream pointed at 127.0.0.1:$1, and a stand-in proxy on 127.0.0.1:$2 +# that answers 502 itself. Prints the console port. +start_console_nginx() { + upstream_port=$1 standin_port=$2 + conf=$(console_nginx_template) + re=$(regex_locations "$conf") + console_port=$(free_port) + { + echo "pid $BATS_TEST_TMPDIR/nginx.pid;" + echo "error_log $BATS_TEST_TMPDIR/error.log;" + echo "events {}" + echo "http {" + echo " access_log off;" + echo " client_body_temp_path $BATS_TEST_TMPDIR; proxy_temp_path $BATS_TEST_TMPDIR;" + echo " fastcgi_temp_path $BATS_TEST_TMPDIR; uwsgi_temp_path $BATS_TEST_TMPDIR; scgi_temp_path $BATS_TEST_TMPDIR;" + echo " server {" + echo " listen 127.0.0.1:$console_port;" + echo " location ~ $re {" + # The test upstream is an address, so the resolver is never asked. + location_body "$conf" "$re" \ + | sed -e 's/\${NGINX_LOCAL_RESOLVERS}/127.0.0.1/' \ + -e "s/^\([[:space:]]*set[[:space:]]*\$observability_proxy[[:space:]]\).*/\1127.0.0.1:$upstream_port;/" + echo " }" + named_location "$conf" "@observability_proxy_unavailable" + echo " }" + echo " server {" + echo " listen 127.0.0.1:$standin_port;" + echo " location / { return 502 'log store unreachable'; }" + echo " }" + echo "}" + } > "$BATS_TEST_TMPDIR/nginx.conf" + nginx -p "$BATS_TEST_TMPDIR" -c "$BATS_TEST_TMPDIR/nginx.conf" \ + || { cat "$BATS_TEST_TMPDIR/error.log" >&2; return 1; } + echo "$console_port" +} + +teardown() { + if [ -f "${BATS_TEST_TMPDIR:-}/nginx.pid" ]; then + kill "$(cat "$BATS_TEST_TMPDIR/nginx.pid")" 2>/dev/null || true + fi +} + +@test "a cluster without the proxy answers the console's not-installed Status, the proxy's own 502 passes through" { + command -v nginx >/dev/null 2>&1 || skip "nginx is not installed" + command -v curl >/dev/null 2>&1 || skip "curl is not installed" + path=/apis/cozystack.io/v1alpha1/namespaces/tenant-foo/logs/select/logsql/query + + # Nothing listens on the upstream port: nginx itself fails the request. + standin=$(free_port) + port=$(start_console_nginx "$(free_port)" "$standin") + out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path") + printf '%s\n' "$out" | head -n 1 | grep -q ' 404' \ + || { echo "FAIL: an unreachable proxy did not answer 404: $out" >&2; false; } + printf '%s\n' "$out" | grep -iq '^content-type: application/json' \ + || { echo "FAIL: the not-installed answer is not JSON: $out" >&2; false; } + body=$(printf '%s\n' "$out" | tail -n 1) + printf '%s' "$body" | python3 -c ' +import json, sys +s = json.load(sys.stdin) +assert s["kind"] == "Status" and s["apiVersion"] == "v1" and s["status"] == "Failure", s +assert s["reason"] == "NotFound" and s["code"] == 404, s +assert s["message"] == "the observability proxy is not available on this cluster", s +' || { echo "FAIL: the not-installed body is not the expected Status: $body" >&2; false; } + # The next nginx gets fresh ports, so the old master need not have + # released its sockets; it is still stopped first to keep one running. + pid=$(cat "$BATS_TEST_TMPDIR/nginx.pid") + kill "$pid" + for _ in $(seq 1 50); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + rm -f "$BATS_TEST_TMPDIR/nginx.pid" + + # The proxy answers 502 itself: the console must see that answer. + standin=$(free_port) + port=$(start_console_nginx "$standin" "$standin") + out=$(curl --silent --show-error --include --max-time 10 "http://127.0.0.1:$port$path") + printf '%s\n' "$out" | head -n 1 | grep -q ' 502' \ + || { echo "FAIL: the proxy's own 502 did not pass through: $out" >&2; false; } + printf '%s\n' "$out" | tail -n 1 | grep -qx 'log store unreachable' \ + || { echo "FAIL: the proxy's own 502 body was replaced: $out" >&2; false; } +} diff --git a/hack/cozyreport.bats b/hack/cozyreport.bats index cc2668071..ee8911080 100644 --- a/hack/cozyreport.bats +++ b/hack/cozyreport.bats @@ -966,6 +966,37 @@ STUB } } +@test "PR E2E jobs clean up between preparation attempts but not after testing" { + tmp="$(mktemp -d)" + mkdir -p "$tmp/bin" + cat >"$tmp/bin/make" <<'STUB' +#!/bin/sh +printf '%s\n' "$*" >>"$CLEANUP_CALLS" +case "$*" in + *prepare-env-container*) + if [ ! -f "$CLEANUP_ATTEMPT" ]; then + touch "$CLEANUP_ATTEMPT" + exit 1 + fi + ;; +esac +STUB + chmod +x "$tmp/bin/make" + for wf in pull-requests e2e-fork; do + workflow="$HACK_DIR/../.github/workflows/$wf.yaml" + yq --unwrapScalar '.jobs.e2e.steps[] | select(.name == "Prepare environment") | .run' "$workflow" >"$tmp/prepare.sh" + CLEANUP_CALLS="$tmp/$wf.calls" CLEANUP_ATTEMPT="$tmp/$wf.attempt" SANDBOX_NAME=test PATH="$tmp/bin:$PATH" bash -eu "$tmp/prepare.sh" + printf '%s\n' 'SANDBOX_NAME=test prepare-env-container' '-C packages/core/testing SANDBOX_NAME=test delete-cluster-container' 'SANDBOX_NAME=test prepare-env-container' >"$tmp/expected" + diff "$tmp/expected" "$tmp/$wf.calls" + scripts="$(yq --unwrapScalar '.jobs.e2e.steps[] | select(.name != "Prepare environment") | .run // ""' "$workflow")" + if printf '%s\n' "$scripts" | grep -Eq 'delete-cluster-container|docker[[:space:]]+(compose.*down|rm)|zpool[[:space:]]+destroy|make.*[[:space:]]delete([[:space:]]|$)'; then + echo "FAIL: $wf destroys runner-local state outside preparation" + false + fi + done + rm -rf "$tmp" +} + @test "the step that runs this collector has a ceiling of its own everywhere" { # Every bound inside the script protects one read. None of them protects the # STEP: the tarball is written on the collector's last line, the upload is the diff --git a/hack/e2e-chainsaw/README.md b/hack/e2e-chainsaw/README.md index 98400bc55..ef5a50db4 100644 --- a/hack/e2e-chainsaw/README.md +++ b/hack/e2e-chainsaw/README.md @@ -26,7 +26,7 @@ A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Cha A parked `chainsaw-test.yaml.disabled` is none of these: the directory is then not a suite, is selected by nothing and cannot be allowlisted. -Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. +Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`, `observability-proxy`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. ## What Chainsaw buys over the BATS suite @@ -63,7 +63,7 @@ Declarative suites assert on `status.conditions` and concrete fields. Inherently ## Running locally -Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`. +Requires a cluster with Cozystack installed and a `tenant-test` namespace (the environment `hack/e2e-install-cozystack.bats` produces). `bucket` and `seaweedfs` additionally need `mc`, `nc`, and `jq` on the host; `kuberture`/`openbao`/`foundationdb`/`securitygroup`/`computeplane`/`kubernetes-latest`/`kubernetes-previous`/`vminstance` need `jq`; `observability-proxy` needs `curl` and `nc`. ```bash # install chainsaw diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 939e8f69a..e6253c06a 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -91,7 +91,7 @@ spec: status: replicas: 1 --- -# ClickHouse Altinity backup/restore E2E — the two backup Tests below sit in +# ClickHouse Altinity backup/restore E2E — the three backup Tests below sit in # this suite alongside the plain `clickhouse` Test above. Chainsaw port of the # intended hack/e2e-apps/backup-clickhouse.bats (cozystack#2600), shaped like the # etcd/mariadb/postgres suites: @@ -107,8 +107,15 @@ spec: # driving the validated example scripts # (examples/backups/clickhouse). Also runs in # CI, in tenant-root. +# clickhouse-3-system-bucket-roundtrip +# — backup and in-place restore through the +# shipped cozy-default BackupClass and the +# platform system bucket, for a release +# whose Pod started before its credentials +# were projected. Runs in CI, in tenant-root. # -# Why the split: the two Tests answer different questions at different cost. The +# Why the split: the first two Tests answer different questions at different +# cost, and the third covers the system-bucket path neither of them takes. The # contracts Test needs no object store, so it stays in the default tenant-test # namespace next to the other app suites and is the seconds-long guard on the # chart wiring the Altinity strategy depends on — the sidecar container/port and @@ -377,3 +384,217 @@ spec: set -eu cd ../../.. NAMESPACE="$NAMESPACE" examples/backups/clickhouse/cleanup.sh +--- +# Backup and in-place restore through the shipped cozy-default BackupClass and +# the platform system bucket, for a ClickHouse whose Pod started before the +# platform projected cozy-backups-creds into its namespace. The sidecar has to +# pick those credentials up while running; until it could, the first BackupJob +# of such a release failed with "no EC2 IMDS role found" and kept failing until +# the Pod was restarted. +# +# Unlike clickhouse-2-backup-roundtrip this targets the system bucket as +# advertised, https://s3.example.org, so system-bucket-roundtrip.sh makes that +# name reachable from tenant-root for the duration of its own step and undoes +# it on exit: see its header for the two shared objects it touches and the +# cert-manager behaviour that relies on. Runs in tenant-root, where the system +# bucket's SeaweedFS and its Ingress controller sit. +# +# The precondition step deletes tenant-root's projected cozy-backups-creds, +# which any BackupJob that ran in tenant-root before this Test leaves there: +# the projector runs for each BackupJob whatever its strategy, and Chainsaw +# does not fix the order of the Tests in a suite, so the earlier one may come +# from this suite or another. It writes the Secret again on the next +# BackupJob, which is the moment under test. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: clickhouse-3-system-bucket-roundtrip +spec: + namespace: tenant-root + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: clickhouse-sysbucket + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: e2e-system-bucket-s3 + - describe: + apiVersion: cert-manager.io/v1 + kind: Certificate + name: seaweedfs-s3-ingress-tls + - describe: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: ingress-seaweedfs-s3 + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: BackupJob + name: sysbucket-backup + - describe: + apiVersion: backups.cozystack.io/v1alpha1 + kind: RestoreJob + name: sysbucket-restore + - podLogs: + selector: app.kubernetes.io/instance=clickhouse-sysbucket + container: clickhouse-backup + tail: 200 + - podLogs: + selector: app=clickhouse-sysbucket-keeper + tail: 100 + steps: + - name: preconditions + try: + # The deletes below wait up to 270s in all (30+30+60+30+30+30+60), so a + # stuck one is named by its kubectl --timeout before the 5m ceiling kills + # the shell with no object named. A new delete needs its own --timeout. + - script: + timeout: 5m + content: | + set -eu + if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then + echo "Secret $NAMESPACE/seaweedfs-s3-ingress-tls already exists; this Test would have to replace it" >&2 + exit 1 + fi + if kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF 'BEGIN e2e-clickhouse-system-bucket'; then + echo "kube-system/coredns still carries the e2e-clickhouse-system-bucket block from an earlier run" >&2 + exit 1 + fi + # Fixed names in a persistent namespace: a leftover Succeeded job + # would satisfy the waits in the run step without a new attempt, and + # a leftover Backup of the BackupJob's name, recording an earlier + # run's archive, fails the new BackupJob. + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s + kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s + # Chainsaw reclaims only what its apply created, so objects a killed + # run left would be patched and kept. The Certificate goes before its + # Secret, which cert-manager issues again while the Certificate exists. + kubectl -n "$NAMESPACE" delete certificate e2e-system-bucket-s3 --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete pod e2e-system-bucket-probe --ignore-not-found --wait=true --timeout=60s + check: + ($error == null): true + + - name: create-certificate-and-probe + try: + - apply: + file: system-bucket-tls.yaml + - assert: + timeout: 2m + resource: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: e2e-system-bucket-s3 + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 3m + resource: + apiVersion: v1 + kind: Pod + metadata: + name: e2e-system-bucket-probe + status: + (conditions[?type == 'Ready']): + - status: "True" + # The Certificate's Secret carries no owner reference, so it outlives the + # Certificate, and cert-manager issues it again while the Certificate + # exists. Chainsaw runs a step's cleanup ops after this step's cleaner has + # deleted the Certificate and waited for it. + cleanup: + - delete: + ref: + apiVersion: v1 + kind: Secret + name: e2e-system-bucket-s3-tls + + - name: create-clickhouse + try: + - apply: + file: clickhouse-system-bucket.yaml + # 11m: above the 600s install timeout cozystack-api gives this release; + # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. + - assert: + timeout: 11m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: clickhouse-sysbucket + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 5m + resource: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: chi-clickhouse-sysbucket-clickhouse-0-0 + status: + readyReplicas: 1 + # The Pod is running with no projected credentials: the state the first + # BackupJob of a fresh release meets. + - error: + timeout: 30s + resource: + apiVersion: v1 + kind: Secret + metadata: + name: cozy-backups-creds + + - name: run + try: + # Inner budgets of system-bucket-roundtrip.sh: + # 280s probe: a 240s deadline for CoreDNS picking up the rewrite and + # the Ingress serving the Test certificate, plus one last 30s + # attempt and its 10s pause + # 480s BackupJob to Succeeded, including the driver's 2m hold + # 480s RestoreJob to Succeeded; no hold by then, the stamp is + # already older than the backup's own 2m wait + # 90s the EXIT trap: Corefile replace plus a 60s Secret delete + # 1330s (22m10s) + # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so + # an inner wait is what fires and names what stalled. + - script: + timeout: 25m + content: | + set -eu + cd ../../.. + NAMESPACE="$NAMESPACE" hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh + check: + ($error == null): true + finally: + # The script's EXIT trap puts the Corefile and the Ingress TLS Secret + # back, but a step ceiling kills the shell before the trap runs. A no-op + # when the trap already did it. + - description: restore the host CoreDNS and the S3 Ingress TLS Secret + script: + timeout: 2m + content: | + set -eu + cd ../../.. + # Chainsaw runs this under sh; the helpers are bash. + NAMESPACE="$NAMESPACE" bash -euo pipefail -c ' + source examples/backups/clickhouse/00-helpers.sh + source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh + restore_shared_state + ' + # Chainsaw did not apply these: the jobs come from the script and the + # Backup from the controller, and deleting a BackupJob reclaims no Backup. + - description: system-bucket round-trip cleanup + script: + timeout: 3m + content: | + set -eu + kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + # The driver names the Backup after the BackupJob; by name it is + # found even when status.backupRef was never written. + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml index b88e7dd6d..fb0979146 100644 --- a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml @@ -8,8 +8,8 @@ # backup.enabled=true with placeholder S3 coordinates is deliberate: the # sidecar runs `clickhouse-backup server`, which starts its HTTP API without # validating S3 (ALLOW_EMPTY_BACKUPS=true, lazy connect — the same reason the -# platform useSystemBucket flow marks the S3 secret keys optional so the Pod -# starts before any bucket is projected). So the Pod becomes Ready and the +# platform useSystemBucket flow can start the Pod before any bucket is +# projected). So the Pod becomes Ready and the # chart<->strategy contract is assertable with no reachable object store. The # real S3 backup/restore round-trip lives in clickhouse-2-backup-roundtrip, # which runs in tenant-root against the in-cluster SeaweedFS. diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml new file mode 100644 index 000000000..0ed1dd4a4 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml @@ -0,0 +1,30 @@ +# ClickHouse on the platform system bucket for the +# clickhouse-3-system-bucket-roundtrip Test. No S3 coordinates are given: the +# sidecar reads them from the clickhouse-backup.yml key the platform projects +# into cozy-backups-creds at the first BackupJob, after this Pod has started. +# backup.endpointCA trusts the SeaweedFS CA that signs the certificate the Test +# serves for s3.example.org. +apiVersion: apps.cozystack.io/v1alpha1 +kind: ClickHouse +metadata: + name: sysbucket +spec: + size: 5Gi + logStorageSize: 1Gi + shards: 1 + replicas: 1 + # The round-trip runs SQL against this server; at nano's 128Mi the first + # query OOM-kills it, so it takes the preset clickhouse-2 runs SQL on. + resourcesPreset: small + resources: {} + backup: + enabled: true + useSystemBucket: true + endpointCA: + name: e2e-system-bucket-s3-tls + key: ca.crt + clickhouseKeeper: + enabled: true + replicas: 1 + size: 1Gi + resourcesPreset: micro diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh new file mode 100755 index 000000000..c1c216389 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# Backup and in-place restore of the `sysbucket` ClickHouse through the +# cozy-default BackupClass, against the platform system bucket. Run from the +# repository root by the clickhouse-3-system-bucket-roundtrip Test, which has +# already started the release with no cozy-backups-creds in its namespace. +# +# The system bucket is addressed as https://s3.example.org, the S3 Ingress host +# the COSI driver advertises. In CI that name does not resolve and its ACME +# certificate is never issued, so for the duration of this script, and only of +# this script, two shared objects are changed and the EXIT trap below puts both +# back: +# +# - the S3 Ingress TLS Secret seaweedfs-s3-ingress-tls is created from the +# Test's own certificate (SeaweedFS CA, s3.example.org). It relies on +# cert-manager leaving a Secret's data alone while the Certificate that +# names it has no completed issuance, verified with cert-manager v1.20.2: +# a fresh ACME order would replace this certificate, and ingress-nginx +# would then serve one the sidecar does not trust, which fails the probe +# below rather than the backup. +# - the host CoreDNS Corefile gets one marked `rewrite name exact` line for +# s3.example.org. The host CoreDNS is Talos-managed and imports no snippet +# files, so the block goes into the live Corefile the way ouroboros writes +# its own; ouroboros is off in CI and no other suite uses this name. +set -euo pipefail + +source examples/backups/clickhouse/00-helpers.sh +source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh + +: "${NAMESPACE:?}" +CLICKHOUSE_NAME=sysbucket +BACKUPJOB_NAME=sysbucket-backup +RESTOREJOB_NAME=sysbucket-restore +CERT_SECRET=e2e-system-bucket-s3-tls +PROBE_POD=e2e-system-bucket-probe +DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local" +# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's +# reload plugin polls every 30s; the rest is the 30s answer cache. +PROBE_TIMEOUT=240 +# Each covers the driver's 2m hold after the credentials change plus the +# strategy Job itself. +JOB_TIMEOUT=480 + +workdir=$(mktemp -d) + +on_exit() { + local rc=$? + set +e + restore_shared_state || rc=1 + rm -rf "$workdir" + exit "$rc" +} + +if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -o name >/dev/null 2>&1; then + log_error "Secret ${NAMESPACE}/${INGRESS_SECRET} already exists; this Test serves its own certificate under that name and will not overwrite one it did not create" + exit 1 +fi +if corefile_has_block; then + log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand" + exit 1 +fi +trap on_exit EXIT + +log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \ + | jq --arg name "$INGRESS_SECRET" '{apiVersion, kind, type, + metadata: {name: $name, labels: {"cozystack.io/e2e-owner": "clickhouse-system-bucket"}}, + data: {"tls.crt": .data["tls.crt"], "tls.key": .data["tls.key"], "ca.crt": .data["ca.crt"]}}' \ + | kubectl -n "$NAMESPACE" create -f - + +log_step "Rewriting ${S3_HOST} to ${DNS_TARGET} in the host CoreDNS" +kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json" +jq -e '.data.Corefile | startswith(".:53 {\n")' "$workdir/coredns.json" >/dev/null \ + || { log_error "kube-system/coredns Corefile does not open with '.:53 {'; refusing to edit it"; exit 1; } +# The resourceVersion read above travels with the replace, so a concurrent +# writer makes it fail instead of being overwritten. +jq --arg block "${DNS_BEGIN}"$'\n'" rewrite name exact ${S3_HOST} ${DNS_TARGET}"$'\n'"${DNS_END}" \ + '.data.Corefile |= sub("^\\.:53 \\{\n"; ".:53 {\n" + $block + "\n")' \ + "$workdir/coredns.json" >"$workdir/coredns-new.json" +kubectl replace -f "$workdir/coredns-new.json" >/dev/null + +log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAMESPACE}" +kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt" +want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \ + | openssl x509 -noout -fingerprint -sha256) +deadline=$((SECONDS + PROBE_TIMEOUT)) +while true; do + # --cacert makes curl verify both the chain and the host name; %{certs} + # prints the chain it was served, leaf first. + if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ + "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \ + <"$workdir/ca.crt" 2>"$workdir/probe.err"); then + got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \ + | openssl x509 -noout -fingerprint -sha256) + if [ "$got" = "$want" ]; then + log_success "${S3_HOST} resolves in ${NAMESPACE} and serves the Test certificate" + break + fi + echo "served ${got}, want ${want}" >"$workdir/probe.err" + fi + if [ "$SECONDS" -ge "$deadline" ]; then + log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")" + kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true + kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true + exit 1 + fi + # TODO(e2e-replace-fixed-timeouts): no condition reports that the host + # CoreDNS answers with the rewrite and the Ingress serves the Test + # certificate, as seen from this namespace; poll until one does. + sleep 10 +done + +# A restarted Pod reads its credentials afresh at start, which is what the +# old environment-based sidecar needed; the round-trip proves the fix only if +# the sidecar that started without them is the one that serves both jobs. +sidecar_identity() { + kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \ + -o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}' +} +sidecar_before=$(sidecar_identity) +[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ + || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } + +# Restore takes the archive this run's Backup recorded; a row no earlier run +# wrote is what shows the data restored is that archive's. +sentinel="before-backup-$(date -u +%Y%m%dT%H%M%SZ)-${RANDOM}" +log_step "Writing the sentinel row ${sentinel}" +clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" +clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, '${sentinel}')" + +log_step "Backing up through cozy-default" +kubectl apply -f - <"$dir/coredns.json" || { rm -rf "$dir"; return 1; } + local begins + begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true) + if [ "$begins" = 0 ]; then + rm -rf "$dir" + return 0 + fi + if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then + log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand" + rm -rf "$dir" + return 1 + fi + jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \ + '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \ + "$dir/coredns.json" >"$dir/coredns-clean.json" + # Callers run this with errexit off, so a refused replace has to return. + if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then + log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand" + rm -rf "$dir" + return 1 + fi + log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns" + rm -rf "$dir" +} + +# Returns non-zero when either object could not be put back. +restore_shared_state() { + local rc=0 + remove_dns_block || rc=1 + # kubectl refuses a name together with a selector, so the label is + # looked up on its own and the name matched in what comes back. + local owned + if ! owned=$(kubectl -n "$NAMESPACE" get secret -l "$OWNER_LABEL" -o name); then + log_error "could not list the Secrets labelled ${OWNER_LABEL} in ${NAMESPACE}; delete ${INGRESS_SECRET} by hand if it carries that label" + rc=1 + elif printf '%s\n' "$owned" | grep -qxF "secret/$INGRESS_SECRET"; then + kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1 + fi + return "$rc" +} diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml new file mode 100644 index 000000000..06203a299 --- /dev/null +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml @@ -0,0 +1,42 @@ +# Certificate and probe Pod for the clickhouse-3-system-bucket-roundtrip Test. +# +# The certificate is signed by the SeaweedFS CA already present in tenant-root, +# so its Secret carries that CA as ca.crt: the ClickHouse release trusts it via +# backup.endpointCA, and the round-trip script copies the same key pair under +# the name the platform's S3 Ingress serves for s3.example.org. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-system-bucket-s3 +spec: + secretName: e2e-system-bucket-s3-tls + dnsNames: + - s3.example.org + issuerRef: + kind: Issuer + name: seaweedfs-ca-issuer +--- +# Resolves and fetches s3.example.org from inside tenant-root, where the +# ClickHouse sidecar runs, so the script can prove the DNS rewrite and the +# served certificate before it spends minutes on a BackupJob. The image is the +# one hack/e2e-install-cozystack.bats pre-pulls; it carries curl but no openssl. +apiVersion: v1 +kind: Pod +metadata: + name: e2e-system-bucket-probe +spec: + restartPolicy: Never + terminationGracePeriodSeconds: 0 + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: probe + image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47 + command: [sleep, "3600"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] diff --git a/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml b/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml new file mode 100644 index 000000000..576f6fa6d --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/chainsaw-test.yaml @@ -0,0 +1,257 @@ +# Namespace logs through the observability proxy, as the console reaches +# them: every request goes through the console nginx (a port-forward to its +# Service), which routes the logs root to the proxy, which authenticates the +# bearer token, asks a SubjectAccessReview and forwards a rebuilt query to +# the tenant's log store. +# +# Two sibling tenants, A (logsa) and B (logsb), both under tenant-test, so +# both resolve to the root log store. A viewer of A reads A's logs and is +# refused B's; parameters and headers a client adds cannot widen the scope +# (the proxy forces the namespace filter and tenant 0:0); a live tail stays +# open and keeps delivering lines past 60s. +# +# Not covered here: the OIDC gatekeeper (oauth2-proxy) in front of nginx, +# because driving a Keycloak login needs a browser; and Fluent Bit writing +# real pod logs, because the root log store is off in e2e and this suite +# seeds its own (log-store.yaml). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: observability-proxy-logs +spec: + catch: + - events: {} + - podLogs: + namespace: cozy-observability-proxy + selector: app.kubernetes.io/name=observability-proxy + tail: 100 + - podLogs: + namespace: tenant-root + selector: app.kubernetes.io/name=e2e-observability-log-store + tail: 100 + - podLogs: + namespace: cozy-dashboard + selector: app.kubernetes.io/name=console + tail: 100 + steps: + - name: proxy-available + try: + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: observability-proxy + namespace: cozy-observability-proxy + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: tenants + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: logsa + spec: {} + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: logsb + spec: {} + # 16m: tenant-rd's 15m install timeout plus the 60s start margin; see + # "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. Waited so + # that cleanup does not uninstall a release whose install still runs. + - assert: + timeout: 16m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: tenant-logsa + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 16m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: tenant-logsb + status: + (conditions[?type == 'Ready']): + - status: "True" + # The label names the store the proxy reads; the seeded store below + # stands in for the root one, so both namespaces must resolve to it. + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-logsa + labels: + namespace.cozystack.io/monitoring: tenant-root + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-logsb + labels: + namespace.cozystack.io/monitoring: tenant-root + + - name: viewer-and-store + try: + # Applying over a real root store would rewrite its Service; refuse to. + - error: + resource: + apiVersion: v1 + kind: Service + metadata: + name: vlselect-generic + namespace: tenant-root + - apply: + file: viewer.yaml + - apply: + file: log-store.yaml + - assert: + timeout: 3m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: e2e-observability-log-store + namespace: tenant-root + status: + (conditions[?type == 'Available']): + - status: "True" + + - name: seed-logs + try: + - script: + # 30s forward + 3 x 15s ingests + 3 x 30s visibility polls, each + # capped on its own, under a 4m op. + timeout: 4m + content: | + set -eu + . ./lib.sh + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT + forward tenant-root vlselect-generic 19471:9471 + ingest tenant-test-logsa e2e-marker-a + ingest tenant-test-logsb e2e-marker-b + # Same namespace, another VictoriaLogs tenant: reachable only if a + # client AccountID header got through. + ingest tenant-test-logsa e2e-marker-account7 7 + # All three must be searchable, or an absence checked later + # proves nothing. + for m in e2e-marker-a:0 e2e-marker-b:0 e2e-marker-account7:7; do + timeout 30 sh -ec "until curl --silent --fail --header 'AccountID: ${m#*:}' '$STORE/select/logsql/query?query=${m%:*}' | grep -q ${m%:*}; do sleep 1; done" + done + + - name: namespace-scope + try: + - script: + # 30s forward + one 30s token request + 10 calls capped at 30s. + timeout: 8m + content: | + set -eu + . ./lib.sh + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT + forward cozy-dashboard cozy-dashboard-console 18080:8080 + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) + auth="Authorization: Bearer $token" + + code=$(call "$LOGS_A/query?query=*") + expect 401 "$code" "no token" + + code=$(call "$LOGS_A/query?query=*" --header "$auth") + expect 200 "$code" "viewer reads A" + grep -q e2e-marker-a "$body" || { echo "FAIL: A's own line is missing" >&2; cat "$body" >&2; exit 1; } + absent e2e-marker-b "viewer reads A" + absent e2e-marker-account7 "viewer reads A" + + code=$(call "$LOGS_B/query?query=*" --header "$auth") + expect 403 "$code" "viewer reads B" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query=*' \ + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}' \ + --data-urlencode 'extra_filters=kubernetes_namespace_name:tenant-test-logsb') + expect 200 "$code" "client extra filters in the query string" + absent e2e-marker-b "client extra filters in the query string" + + code=$(call "$LOGS_A/query" --header "$auth" \ + --data-urlencode 'query=*' \ + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}') + expect 200 "$code" "client extra filters in a POST form" + absent e2e-marker-b "client extra filters in a POST form" + + code=$(call "$LOGS_A/query?query=*" --header "$auth" --header "AccountID: 7" --header "ProjectID: 0") + expect 200 "$code" "client AccountID header" + absent e2e-marker-account7 "client AccountID header" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query={kubernetes_namespace_name="tenant-test-logsb"}') + expect 200 "$code" "B's stream named in the query" + absent e2e-marker-b "B's stream named in the query" + + code=$(call "$LOGS_A/query" --header "$auth" --get \ + --data-urlencode 'query=options(ignore_global_time_filter=true) *') + expect 400 "$code" "query options" + + # nginx routes on the normalised path, which is B's logs root, and + # forwards the raw one; the proxy must refuse it, not read B. + code=$(call "$LOGS_A/../../../../tenant-test-logsb/logs/select/logsql/query?query=*" --path-as-is --header "$auth") + expect 400 "$code" "dot segments toward B" + + code=$(call "$LOGS_A/delete?query=*" --header "$auth") + expect 404 "$code" "an endpoint outside the allowlist" + + - name: tail-outlives-a-minute + try: + - script: + # 2 x 30s forwards + 30s token + 3 x 15s ingests + the 70s hold + + # the 30s delivery poll; the tail itself is cut at 150s. + timeout: 6m + content: | + set -eu + . ./lib.sh + tail_out=$(mktemp) + trap 'kill "${tail_pid:-}" 2>/dev/null || true; for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body" "$tail_out"' EXIT + forward tenant-root vlselect-generic 19471:9471 + forward cozy-dashboard cozy-dashboard-console 18080:8080 + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) + + # The tail is not subscribed yet when the next ingest runs, so it + # replays the last minute to include the early line. + timeout 150 curl --silent --no-buffer --header "Authorization: Bearer $token" \ + "$CONSOLE$LOGS_A/tail?query=*&start_offset=1m" > "$tail_out" & + tail_pid=$! + ingest tenant-test-logsa e2e-tail-early + ingest tenant-test-logsb e2e-tail-b + # TODO(e2e-replace-fixed-timeouts): this hold stands in for a + # duration assertion the harness lacks; the property under test is + # that a tail survives a minute with the client sending nothing. + sleep 70 + if ! kill -0 "$tail_pid" 2>/dev/null; then + echo "FAIL: the tail closed before 70s" >&2 + cat "$tail_out" >&2 + exit 1 + fi + ingest tenant-test-logsa e2e-tail-late + # vlselect re-reads every second, five seconds behind now. + timeout 30 sh -ec "until grep -q e2e-tail-late '$tail_out'; do sleep 1; done" || { + echo "FAIL: a line written at 70s did not reach the open tail" >&2 + cat "$tail_out" >&2 + exit 1 + } + grep -q e2e-tail-early "$tail_out" || { echo "FAIL: the early line is missing from the tail" >&2; exit 1; } + if grep -q e2e-tail-b "$tail_out"; then + echo "FAIL: the tail of A delivered B's line" >&2 + exit 1 + fi diff --git a/hack/e2e-chainsaw/observability-proxy/lib.sh b/hack/e2e-chainsaw/observability-proxy/lib.sh new file mode 100644 index 000000000..9601cb826 --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/lib.sh @@ -0,0 +1,58 @@ +# shellcheck shell=sh +# Shared by the script steps of this suite, whose cwd is this directory. +# Every call carries its own cap so a hang fails at a known budget rather +# than at the op ceiling (docs/agents/e2e-testing.md, section 8). + +k() { timeout 30 kubectl "$@"; } + +STORE=http://localhost:19471 +CONSOLE=http://localhost:18080 +# shellcheck disable=SC2034 # read by the steps that source this file +LOGS_A=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsa/logs/select/logsql +# shellcheck disable=SC2034 +LOGS_B=/apis/cozystack.io/v1alpha1/namespaces/tenant-test-logsb/logs/select/logsql + +# forward :. The caller's EXIT trap +# kills every pid in pf_pids; the forward gets 30s to start listening. +forward() { + kubectl port-forward -n "$1" "service/$2" "$3" >/dev/null 2>&1 & + pf_pids="${pf_pids:-} $!" + timeout 30 sh -ec "until nc -z localhost ${3%%:*}; do sleep 1; done" +} + +# ingest [account]: one entry shaped as Fluent Bit +# writes it, with kubernetes_namespace_name as a stream field. +ingest() { + printf '{"kubernetes_namespace_name":"%s","log":"%s"}\n' "$1" "$2" | + timeout 15 curl --silent --show-error --fail --data-binary @- \ + --header 'Content-Type: application/json' --header "AccountID: ${3:-0}" --header "ProjectID: 0" \ + "$STORE/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log" +} + +# call [curl args...]: a request through the console nginx, the way +# the browser sends it. Prints the status; the body lands in $body. +body=$(mktemp) +call() { + path=$1 + shift + timeout 30 curl --silent --show-error --output "$body" --write-out '%{http_code}' "$@" "$CONSOLE$path" +} + +# expect +expect() { + if [ "$1" != "$2" ]; then + echo "FAIL: $3: HTTP $2, want $1" >&2 + cat "$body" >&2 + exit 1 + fi + echo "ok: $3: HTTP $2" +} + +# absent : the last body must not carry the marker. +absent() { + if grep -q "$1" "$body"; then + echo "FAIL: $2: the response carries $1, which is outside the scope" >&2 + cat "$body" >&2 + exit 1 + fi +} diff --git a/hack/e2e-chainsaw/observability-proxy/log-store.yaml b/hack/e2e-chainsaw/observability-proxy/log-store.yaml new file mode 100644 index 000000000..491dc667c --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/log-store.yaml @@ -0,0 +1,71 @@ +# A single-node VictoriaLogs under the name the proxy dials for a namespace +# whose logs belong to the root tenant: vlselect-generic.tenant-root, port +# 9471. The e2e install leaves root monitoring off, so the real one does not +# exist; the suite seeds this one itself instead of waiting for Fluent Bit, +# which keeps the test about the proxy's scope rather than log collection. +# The version is the one victoria-metrics-operator v0.68.4 deploys. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: e2e-observability-log-store + namespace: tenant-root +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: e2e-observability-log-store + template: + metadata: + labels: + app.kubernetes.io/name: e2e-observability-log-store + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: victoria-logs + image: docker.io/victoriametrics/victoria-logs:v1.47.0 + args: + - -httpListenAddr=:9471 + - -storageDataPath=/storage + - -retentionPeriod=1d + ports: + - name: http + containerPort: 9471 + readinessProbe: + httpGet: + path: /health + port: http + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: storage + mountPath: /storage + volumes: + - name: storage + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: vlselect-generic + namespace: tenant-root +spec: + selector: + app.kubernetes.io/name: e2e-observability-log-store + ports: + - name: http + port: 9471 + targetPort: http diff --git a/hack/e2e-chainsaw/observability-proxy/viewer.yaml b/hack/e2e-chainsaw/observability-proxy/viewer.yaml new file mode 100644 index 000000000..0563ad4a0 --- /dev/null +++ b/hack/e2e-chainsaw/observability-proxy/viewer.yaml @@ -0,0 +1,22 @@ +# The view level of tenant A and nothing else: a ServiceAccount of tenant A +# bound to cozy:tenant:view in A's namespace only. Its token is a real bearer +# token, so the proxy's TokenReview and SubjectAccessReview run unmocked. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: logs-viewer + namespace: tenant-test-logsa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: logs-viewer + namespace: tenant-test-logsa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:tenant:view +subjects: +- kind: ServiceAccount + name: logs-viewer + namespace: tenant-test-logsa diff --git a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml index 170c0ce36..bc0bc1c98 100644 --- a/hack/e2e-chainsaw/postgres/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/postgres/chainsaw-test.yaml @@ -347,3 +347,148 @@ spec: set -eu cd ../../.. NAMESPACE="$NAMESPACE" examples/backups/postgres/cleanup.sh +--- +# A system database that a version before the refusal took over is handed back +# by the next init-job run, and only what this chart marked changes hands. +# +# The helm-unittest and bats suites check the rendered SQL against a fake psql; +# only a real PostgreSQL can say that the predicates select what they claim. The +# state an older version left is rebuilt by hand as the superuser, since the +# chart now refuses to produce it: +# * postgres is commented as managed and owned by a postgres_admin role that +# carries the cozystack:managed mark, as the chart left it: it must come back +# to the superuser, and the role must be dropped; +# * template1 is commented as managed, but its template1_admin carries only the +# role comment, which a CREATEROLE holder can forge, and not the mark: the +# comment must be cleared and everything that role owns or was granted must +# stay as it is, in template1 and in a database cloned from it. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: postgres-3-system-databases +spec: + catch: + - events: {} + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: postgres-sysdb + - podLogs: + selector: job-name=postgres-sysdb-init-job + tail: 200 + steps: + - name: create-postgres + try: + - apply: + file: postgres-sysdb.yaml + # 11m: above the 600s install timeout cozystack-api gives this release; + # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. + - assert: + timeout: 11m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: postgres-sysdb + status: + (conditions[?type == 'Ready']): + - status: "True" + - assert: + timeout: 2m + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: postgres-sysdb-init-job + status: + (conditions[?type == 'Complete']): + - status: "True" + + - name: rebuild-taken-over-system-databases + try: + - script: + timeout: 3m + content: | + set -eu + pod=$(kubectl -n "$NAMESPACE" get pod \ + -l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1) + [ -n "$pod" ] || { echo "no primary pod for postgres-sysdb" >&2; exit 1; } + sql() { kubectl -n "$NAMESPACE" exec -i "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -d "$1"; } + sql postgres <<'SQL' + CREATE ROLE postgres_admin NOLOGIN; + COMMENT ON ROLE postgres_admin IS 'role managed by helm'; + GRANT "cozystack:managed" TO postgres_admin WITH INHERIT FALSE, SET FALSE; + COMMENT ON DATABASE postgres IS 'database managed by helm'; + ALTER DATABASE postgres OWNER TO postgres_admin; + CREATE ROLE template1_admin NOLOGIN; + COMMENT ON ROLE template1_admin IS 'role managed by helm'; + CREATE ROLE template1_readonly NOLOGIN; + COMMENT ON DATABASE template1 IS 'database managed by helm'; + ALTER DATABASE template1 OWNER TO template1_admin; + SQL + sql template1 <<'SQL' + ALTER SCHEMA public OWNER TO template1_admin; + CREATE TABLE public.kept (id int); + ALTER TABLE public.kept OWNER TO template1_admin; + GRANT SELECT ON public.kept TO template1_readonly; + SQL + sql postgres <<'SQL' + CREATE DATABASE sysdb_clone TEMPLATE template1; + SQL + # The next run must be a new Job: remove the completed one so the + # assert below cannot match it. + kubectl -n "$NAMESPACE" delete job postgres-sysdb-init-job --wait=true + # Declaring a user is enough for an upgrade, and runs the init-job. + kubectl -n "$NAMESPACE" patch postgres.apps.cozystack.io sysdb --type merge \ + -p '{"spec":{"users":{"sysdbuser":{},"sysdbprobe":{}}}}' + check: + ($error == null): true + - assert: + timeout: 11m + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: postgres-sysdb-init-job + status: + (conditions[?type == 'Complete']): + - status: "True" + + - name: verify-release + try: + - script: + timeout: 2m + content: | + set -eu + pod=$(kubectl -n "$NAMESPACE" get pod \ + -l cnpg.io/cluster=postgres-sysdb,cnpg.io/instanceRole=primary -o name | head -n1) + q() { kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- psql -v ON_ERROR_STOP=1 -U postgres -tA -d "$1" -c "$2"; } + fail=0 + expect() { + got=$(q "$1" "$3") + if [ "$got" != "$4" ]; then + echo "FAIL: $2: got '$got', want '$4'" >&2 + fail=1 + fi + } + comment="SELECT coalesce(shobj_description(oid, 'pg_database'), '') FROM pg_database WHERE datname = " + owner="SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = " + expect postgres "postgres comment cleared" "$comment'postgres'" '' + expect postgres "postgres back to the superuser" "$owner'postgres'" postgres + expect postgres "marked postgres_admin dropped" "SELECT count(*) FROM pg_roles WHERE rolname = 'postgres_admin'" 0 + expect postgres "template1 comment cleared" "$comment'template1'" '' + expect postgres "unmarked template1_admin still exists" "SELECT count(*) FROM pg_roles WHERE rolname = 'template1_admin'" 1 + expect postgres "template1 still owned by the unmarked role" "$owner'template1'" template1_admin + for db in template1 sysdb_clone; do + expect "$db" "$db public still owned by the unmarked role" \ + "SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = 'public'" template1_admin + expect "$db" "$db kept table still owned by the unmarked role" \ + "SELECT tableowner FROM pg_tables WHERE schemaname = 'public' AND tablename = 'kept'" template1_admin + expect "$db" "$db grant to the unmarked readonly role kept" \ + "SELECT has_table_privilege('template1_readonly', 'public.kept', 'SELECT')" t + done + exit "$fail" + check: + ($error == null): true +# The Postgres CR was applied by chainsaw; deleting it removes the whole +# cluster, the databases and roles built above included. diff --git a/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml b/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml new file mode 100644 index 000000000..975dc6a51 --- /dev/null +++ b/hack/e2e-chainsaw/postgres/postgres-sysdb.yaml @@ -0,0 +1,18 @@ +apiVersion: apps.cozystack.io/v1alpha1 +kind: Postgres +metadata: + name: sysdb +spec: + external: false + size: 1Gi + replicas: 1 + storageClass: "" + quorum: + minSyncReplicas: 0 + maxSyncReplicas: 0 + users: + sysdbuser: {} + databases: + appdb: {} + resources: {} + resourcesPreset: "nano" diff --git a/hack/label-workflows-contract.bats b/hack/label-workflows-contract.bats index 32454ffe4..5e80293ac 100644 --- a/hack/label-workflows-contract.bats +++ b/hack/label-workflows-contract.bats @@ -214,7 +214,7 @@ assert_labels_defined() { count="$(stale_exempt_list exempt-issue-labels | wc -l | tr -d ' ')" [ "$count" -eq 12 ] count="$(stale_exempt_list exempt-pr-labels | wc -l | tr -d ' ')" - [ "$count" -eq 4 ] + [ "$count" -eq 2 ] # The renewal path: frozen exempts both; an accepted issue or a held PR is # deliberate, not stale. diff --git a/hack/observability-proxy-seed.bats b/hack/observability-proxy-seed.bats new file mode 100644 index 000000000..26caf2091 --- /dev/null +++ b/hack/observability-proxy-seed.bats @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +prepare_seed_capture() { + strict_setup + seed_tmp=$(mktemp -d) + seed_old_path=$PATH + mkdir -p "$seed_tmp/bin" + export SEED_CAPTURE="$seed_tmp/seed" + export PATH="$seed_tmp/bin:$PATH" + cat > "$seed_tmp/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -eu +printf '%s\n' "$@" > "$SEED_CAPTURE.args" +cat > "$SEED_CAPTURE.body" +if ! grep --fixed-strings --line-regexp --quiet 'Content-Type: application/json' "$SEED_CAPTURE.args"; then + echo 'seed request is missing its JSON content type' >&2 + exit 22 +fi +EOF + chmod +x "$seed_tmp/bin/curl" +} + +assert_seed_request() { + local account=$1 + prepare_seed_capture + # shellcheck source=e2e-chainsaw/observability-proxy/lib.sh + . "$REPO_ROOT/hack/e2e-chainsaw/observability-proxy/lib.sh" + if [ "$account" = 0 ]; then + ingest tenant-test-logsa e2e-marker-a + else + ingest tenant-test-logsa e2e-marker-a "$account" + fi + grep --fixed-strings --line-regexp --quiet "AccountID: $account" "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet 'ProjectID: 0' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet 'http://localhost:19471/insert/jsonline?_stream_fields=kubernetes_namespace_name&_msg_field=log' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet -- '--data-binary' "$SEED_CAPTURE.args" + grep --fixed-strings --line-regexp --quiet '@-' "$SEED_CAPTURE.args" + printf '%s\n' '{"kubernetes_namespace_name":"tenant-test-logsa","log":"e2e-marker-a"}' > "$SEED_CAPTURE.expected" + cmp "$SEED_CAPTURE.expected" "$SEED_CAPTURE.body" + rm "$body" "$SEED_CAPTURE.args" "$SEED_CAPTURE.body" "$SEED_CAPTURE.expected" "$seed_tmp/bin/curl" + rmdir "$seed_tmp/bin" "$seed_tmp" + export PATH="$seed_old_path" +} + +@test "observability seeds send JSON without consuming the body as form data" { + assert_seed_request 0 +} + +@test "observability seeds preserve an explicit VictoriaLogs account" { + assert_seed_request 7 +} diff --git a/hack/postgres-release-system-databases.bats b/hack/postgres-release-system-databases.bats new file mode 100644 index 000000000..688f20df5 --- /dev/null +++ b/hack/postgres-release-system-databases.bats @@ -0,0 +1,261 @@ +#!/usr/bin/env bats +# Behavioural tests for the "release system databases" pass of the Postgres +# init script: +# +# packages/apps/postgres/templates/init-script.yaml +# +# The pass is cut out of the rendered script and RUN against a fake psql that +# records which database each session connects to. The helm-unittest suite in +# packages/apps/postgres/tests/system_databases_test.yaml matches the SQL text, +# and stayed green when the template0 guard was inverted: the rendered pass then +# connected to template0, which accepts no connection, and skipped postgres and +# template1, which are the two it exists to hand back. +# +# What is pinned here: +# * no session ever connects to template0; +# * postgres and template1 are each connected to and released; +# * every taken-over system database, template0 included, has its comment +# cleared; +# * releasing template1 visits every other connectable database, for the +# trigger it passed on to them, and a failed lookup of those databases +# stops the pass before the comment that selects template1 is cleared; +# * the session that releases a system database hands its public schema +# and the database itself back only from a marked _admin or +# _readonly owner (or cozystack:orphaned), before the reassignment, +# an inherited public only from a marked template1_admin, and touches +# only the roles that carry the +# "cozystack:managed" mark, not a role that merely has one of the names. +# +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper + +PG_CHART=packages/apps/postgres + +# render_pass +# Extract the release pass, from its banner to the next one, out of the rendered +# init script. The non-empty check must be an early `return 1`: hack/cozytest.sh +# rewrites every line matching ^}$ into `return 0` plus `}`, helpers included, +# so a check left in last-command position has its status discarded. +render_pass() { + helm template pg-test "$PG_CHART" --namespace tenant-test --set-json '_cluster={"scheduling":{}}' 2>/dev/null \ + | yq eval 'select(.metadata.name == "pg-test-init-script") | .stringData["init.sh"]' - \ + | awk '/^echo "== release system databases"/ { f = 1 } /^echo "== delete databases"/ { f = 0 } f' > "$1" + grep -q 'SYSTEM_DBS=' "$1" || return 1 +} + +# write_fake_psql +# Emit a fake psql. A `-c` query answers from $SYSTEM_DBS for the system +# database lookup and from $OTHER_DBS for the connectable-database lookup, +# which exits with $CLONE_LOOKUP_FAILS instead when that is set. A +# session fed on stdin logs "connect " ("connect -" without --dbname) +# followed by its first line and every -v db= value, to $PLOG, and its whole +# input under a "-- session db=" header to $PLOG.sql. +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. +write_fake_psql() { + cat > "$1/psql" <<'PEOF' +#!/bin/sh +dbname=- query= vdb= +while [ $# -gt 0 ]; do + case "$1" in + --dbname) dbname=$2; shift ;; + -c) query=$2; shift ;; + -v) case "$2" in db=*) vdb=${2#db=} ;; esac; shift ;; + esac + shift +done +if [ -n "$query" ]; then + case "$query" in + *"d.datname IN ('postgres', 'template0', 'template1')"*) printf '%s\n' $SYSTEM_DBS ;; + *datallowconn*) + if [ -n "${CLONE_LOOKUP_FAILS:-}" ]; then + echo "fake psql: connectable-database lookup failed" >&2 + exit "$CLONE_LOOKUP_FAILS" + fi + printf '%s\n' $OTHER_DBS ;; + *) echo "fake psql: unexpected query: $query" >&2; exit 64 ;; + esac + exit 0 +fi +sql=$(cat) +echo "connect $dbname db=$vdb $(printf '%s\n' "$sql" | head -n 1)" >> "$PLOG" +printf -- '-- session %s db=%s\n%s\n' "$dbname" "$vdb" "$sql" >> "$PLOG.sql" +PEOF + chmod +x "$1/psql" +} + +# run_pass +# Run the extracted pass with the fake psql first on PATH; status in $rc. +run_pass() { + export PLOG="$1/calls" + : > "$PLOG" + rc=0 + PATH="$1:$PATH" sh -e "$1/pass.sh" > "$1/out" 2> "$1/err" || rc=$? +} + +@test "a render that produces no release pass fails instead of passing as a no-op" { + tmp=$(mktemp -d) + if PG_CHART=packages/apps/no-such-chart render_pass "$tmp/pass.sh"; then + echo "FAIL: render_pass reported success for a chart that does not exist" + false + fi + rm -rf "$tmp" +} + +@test "release pass connects to postgres and template1, never to template0" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres template0 template1" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q '^connect template0 ' "$tmp/calls"; then + echo "FAIL: the pass connected to template0, which accepts no connection" + cat "$tmp/calls" + false + fi + for db in postgres template1; do + if ! grep -q "^connect $db db=$db DROP EVENT TRIGGER IF EXISTS trigger_auto_grant;" "$tmp/calls"; then + echo "FAIL: $db was not connected to and released" + cat "$tmp/calls" + false + fi + done + for db in postgres template0 template1; do + if ! grep -q "^connect - db=$db COMMENT ON DATABASE :\"db\" IS NULL;" "$tmp/calls"; then + echo "FAIL: the comment on $db was not cleared" + cat "$tmp/calls" + false + fi + done + rm -rf "$tmp" +} + +@test "release session hands public back and touches only marked roles" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres" OTHER_DBS="postgres" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + awk '/^-- session / { f = ($0 == "-- session postgres db=postgres") } f' "$tmp/calls.sql" > "$tmp/release.sql" + # Each statement must end on its whole predicate: a clause appended after the + # mark (an OR, say) would select an unmarked role again. + reassign_where="WHERE EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid JOIN pg_roles u ON u.oid = m.member WHERE g.rolname = 'cozystack:managed' AND u.rolname = r)\\gexec" + owner_marked="(o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\\gexec" + public_where="WHERE n.nspname = 'public' AND $owner_marked" + database_where="WHERE d.datname = :'db' AND $owner_marked" + public_line=$(grep -n 'ALTER SCHEMA public OWNER TO' "$tmp/release.sql" | head -1) + database_line=$(grep -n 'ALTER DATABASE' "$tmp/release.sql" | head -1) + reassign_line=$(grep -n 'REASSIGN OWNED BY' "$tmp/release.sql" | head -1) + if [ "${public_line##*"$public_where"}" != "" ] || [ -z "$public_line" ]; then + echo "FAIL: the release session does not hand public back only from a marked _admin or _readonly owner" + cat "$tmp/release.sql" + false + fi + if [ "${database_line##*"$database_where"}" != "" ] || [ -z "$database_line" ]; then + echo "FAIL: the release session does not take the database back only from a marked owner" + cat "$tmp/release.sql" + false + fi + if [ "${reassign_line##*"$reassign_where"}" != "" ] || [ -z "$reassign_line" ]; then + echo "FAIL: the release session does not reassign only roles carrying the cozystack:managed mark" + cat "$tmp/release.sql" + false + fi + # public and the database must be handed back before REASSIGN OWNED would + # move them to cozystack:orphaned. + if [ "${public_line%%:*}" -ge "${reassign_line%%:*}" ] || [ "${database_line%%:*}" -ge "${reassign_line%%:*}" ]; then + echo "FAIL: public or the database is handed back after the reassignment has already moved it" + cat "$tmp/release.sql" + false + fi + rm -rf "$tmp" +} + +@test "release pass of template1 visits every other connectable database" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="template1" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + for db in postgres app; do + if ! grep -q "^connect $db db= SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant'" "$tmp/calls"; then + echo "FAIL: $db was not checked for the trigger inherited from template1" + cat "$tmp/calls" + false + fi + done + # An inherited public changes hands only from a template1_admin the chart + # marked; an owner that merely has the name keeps it. + inherited_where="WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid)\\gexec" + inherited_line=$(awk '/^-- session / { f = ($0 == "-- session app db=") } f' "$tmp/calls.sql" | grep 'ALTER SCHEMA public OWNER TO' | head -1) + if [ -z "$inherited_line" ] || [ "${inherited_line##*"$inherited_where"}" != "" ]; then + echo "FAIL: an inherited public is handed back without checking that a marked template1_admin owns it" + cat "$tmp/calls.sql" + false + fi + rm -rf "$tmp" +} + +@test "release pass of template1 stops before clearing its comment when the clone lookup fails" { + # Clearing the comment is what stops later runs from selecting template1, so + # a lookup failure that let the pass go on would skip the clone repair for + # good. + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="template1" OTHER_DBS="postgres app" CLONE_LOOKUP_FAILS=73 + run_pass "$tmp" + unset CLONE_LOOKUP_FAILS + + if [ "$rc" -ne 73 ]; then + echo "FAIL: the pass exited $rc instead of stopping on the failed lookup (73)" + cat "$tmp/out" "$tmp/err" "$tmp/calls" + false + fi + if grep -q 'COMMENT ON DATABASE' "$tmp/calls"; then + echo "FAIL: the comment on template1 was cleared although its clones were never visited" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "release pass of postgres alone leaves the other databases alone" { + tmp=$(mktemp -d) + write_fake_psql "$tmp" + render_pass "$tmp/pass.sh" + export SYSTEM_DBS="postgres" OTHER_DBS="postgres app" + run_pass "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: the pass exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q '^connect app ' "$tmp/calls"; then + echo "FAIL: the pass visited app although template1 was not taken over" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} diff --git a/hack/pr-labeler-contract.bats b/hack/pr-labeler-contract.bats index 2c2daa140..8c8633f7d 100644 --- a/hack/pr-labeler-contract.bats +++ b/hack/pr-labeler-contract.bats @@ -23,9 +23,12 @@ need_node() { return 1 } -# The `script: |` block, de-indented to plain JavaScript. +# The `script: |` block, de-indented to plain JavaScript. The block ends at the +# first non-blank line indented less than its body, where the next job starts. extract_script() { - awk 'f { print substr($0, 13) } /^ +script: \|$/ { f = 1 }' "$WF" > "$1" + awk 'f && length($0) && substr($0, 1, 12) != " " { exit } + f { print substr($0, 13) } + /^ +script: \|$/ { f = 1 }' "$WF" > "$1" [ -s "$1" ] } diff --git a/hack/release-changelog-contract.bats b/hack/release-changelog-contract.bats index 44d0925e9..5938b77f2 100644 --- a/hack/release-changelog-contract.bats +++ b/hack/release-changelog-contract.bats @@ -570,10 +570,11 @@ code_lines() { grep -vE '^[[:space:]]*#'; } exit 1 } - # changelog.md sends the agent to run the docs/release.md checks and the - # validator on its own output, by their working-tree paths. Without these two - # copies it runs the rc tag's versions, which predate every check added since. - for f in docs/release.md hack/validate-changelog.sh; do + # changelog.md sends the agent to run the docs/release/changelogs.md checks + # and the validator on its own output, by their working-tree paths. Without + # these two copies it runs the rc tag's versions, which predate every check + # added since. + for f in docs/release/changelogs.md hack/validate-changelog.sh; do printf '%s\n' "$overlay" | code_lines | grep -qF "cp .release-tooling/$f $f" || { echo "The refresh step does not overlay $f from the tooling checkout, so the" >&2 echo "agent following docs/agents/changelog.md section 9 runs the rc tag's copy." >&2 @@ -581,23 +582,26 @@ code_lines() { grep -vE '^[[:space:]]*#'; } } done - # docs/release.md sits under neither sparse-checkout path. It reaches - # .release-tooling only because cone mode also checks out the files of every - # parent directory of a listed one; without it the copy above fails the job. + # An rc tagged before the release docs were split has no docs/release/ + # directory, and cp does not create one. + printf '%s\n' "$overlay" | code_lines | grep -qE '^[[:space:]]+mkdir -p docs/release$' || { + echo "The refresh step does not create docs/release before copying into it, so" >&2 + echo "the copy fails on every rc tagged before the release docs were split." >&2 + exit 1 + } + + # The tooling checkout has to list both directories the copies read from. tooling="$(printf '%s\n' "$block" | awk ' /^ - name: Checkout release tooling from the dispatch ref$/ { inside = 1; next } /^ - name: / { inside = 0 } inside' | code_lines)" - printf '%s\n' "$tooling" | grep -qE '^[[:space:]]+docs/agents$' || { - echo "The tooling checkout no longer lists docs/agents, which is what carries" >&2 - echo "docs/release.md into .release-tooling." >&2 - exit 1 - } - if printf '%s\n' "$tooling" | grep -qE 'sparse-checkout-cone-mode:[[:space:]]*false'; then - echo "The tooling checkout turned cone mode off, which drops docs/release.md" >&2 - echo "from .release-tooling and fails the refresh step's copy." >&2 - exit 1 - fi + for d in docs/agents docs/release; do + printf '%s\n' "$tooling" | grep -qE "^[[:space:]]+$d\$" || { + echo "The tooling checkout no longer lists $d, so .release-tooling lacks the" >&2 + echo "file the refresh step copies from it." >&2 + exit 1 + } + done # Every job that runs release tooling against an older checkout must source it # from the dispatch ref. A script fetched by a script that is itself missing diff --git a/hack/release-freeze-contract.bats b/hack/release-freeze-contract.bats index 197734a2f..71e907f60 100644 --- a/hack/release-freeze-contract.bats +++ b/hack/release-freeze-contract.bats @@ -1,28 +1,22 @@ #!/usr/bin/env bats -# Contract for the rc freeze, the backport target resolution that depends on it, -# the triggering and concurrency rules that decide whether a backport runs at -# all, and the agreement between those rules and cmd/backport-audit, which -# gates a cut by reproducing them. Like promote-gate-contract.bats, these tests pin executable/structural -# workflow lines rather than prose: a commented-out gate or a guard demoted to a -# comment must never satisfy the contract. +# Contract for the rc freeze and the release-line build it dispatches. Like +# promote-gate-contract.bats, these tests pin executable/structural workflow +# lines rather than prose: a commented-out gate or a guard demoted to a comment +# must never satisfy the contract. # -# Neither workflow can be exercised by a PR lane. cut-prerelease.yaml runs only -# on workflow_dispatch from main or a release line, and its whole purpose is a -# one-way, write-once side effect — the first vX.Y.0-rc.N freezes the line, and -# there is no dry-run mode and no undo short of deleting the branch by hand. -# backport.yaml runs only on a merged main-targeted PR. So every invariant below -# is one that first reports a slip at a real release, on the one commit nobody -# wants to be debugging: getting the freeze condition wrong re-opens a frozen -# line silently, and getting the comparator wrong aims every backport at the -# wrong branch. Cheap to pin here, expensive to discover there. +# No PR lane exercises them. cut-prerelease.yaml runs only on workflow_dispatch +# from main or a release line, and its whole purpose is a one-way, write-once +# side effect — the first vX.Y.0-rc.N freezes the line, and there is no dry-run +# mode and no undo short of deleting the branch by hand. So every invariant +# below is one that first reports a slip at a real release, on the one commit +# nobody wants to be debugging: getting the freeze condition wrong re-opens a +# frozen line silently. Cheap to pin here, expensive to discover there. load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CUT="$REPO_ROOT/.github/workflows/cut-prerelease.yaml" -BACKPORT="$REPO_ROOT/.github/workflows/backport.yaml" -AUDIT="$REPO_ROOT/cmd/backport-audit/main.go" # Strip YAML comments. POSIX `grep` only: the unit-test runner has no ripgrep. # grep exits 1 when nothing is selected (legitimate for an all-comment block) @@ -44,24 +38,6 @@ code_lines() { [ "$rc" -le 1 ] } -# Strip JavaScript comments. backport.yaml's logic lives in a github-script -# `script: |` block, where comments start with `//` and survive code_lines -# untouched — and that block's comments name the very APIs these tests assert -# are gone, so filtering on `#` alone would let a comment satisfy a pin. -script_lines() { - local rc=0 - grep -v '^[[:space:]]*//' || rc=$? - [ "$rc" -le 1 ] -} - -# Body of one job, up to the next top-level job key. -job_block() { - awk -v job=" $1:" ' - $0 == job { inside = 1; next } - /^ [a-z0-9_-]+:$/ { inside = 0 } - inside' "$2" -} - # Body of one `- name:` step, up to the next step at the same indent. step_block() { awk -v want=" - name: $1" ' @@ -78,7 +54,6 @@ step_line() { @test "workflows under contract exist" { [ -f "$CUT" ] - [ -f "$BACKPORT" ] } # ── the freeze condition ───────────────────────────────────────────────────── @@ -274,390 +249,3 @@ step_line() { [ -n "$guard" ] && [ -n "$checkout" ] [ "$guard" -lt "$checkout" ] } - -# ── backport target resolution ─────────────────────────────────────────────── -# Both assertions below run over script_lines: this logic sits in a -# github-script block whose comments discuss getLatestRelease and min-1 by name. - -@test "backport targets come from enumerated branches, not the latest release" { - block="$(job_block prepare "$BACKPORT")" - [ -n "$block" ] - - # Enumerate real branches. getLatestRelease returns the newest published - # STABLE, which throughout a freeze window still names the PREVIOUS line — so - # a `backport` on a fix for the release being stabilised would land on the - # wrong branch and miss the release it was written for. - printf '%s\n' "$block" | script_lines | grep -qF 'github.paginate(github.rest.repos.listBranches' - count="$(printf '%s\n' "$block" | script_lines | grep -cF 'getLatestRelease' || true)" - [ "${count:-0}" -eq 0 ] - - # Only branches matching release-. qualify, so per-release - # staging branches (release-1.6.1) and rc branches never become targets. - printf '%s\n' "$block" | script_lines | grep -qF 'release-(\d+)\.(\d+)$' - - # previous is the second-newest EXISTING line, never an arithmetic guess. The - # old min-1 derivation named a non-existent branch whenever a minor was - # skipped. - printf '%s\n' "$block" | script_lines | grep -qF 'lines.length > 1 ? lines[1].name' - count="$(printf '%s\n' "$block" | script_lines | grep -cE 'parseInt\(min\)[[:space:]]*-[[:space:]]*1|min[[:space:]]*-[[:space:]]*1' || true)" - [ "${count:-0}" -eq 0 ] - - # Asking for a previous line when only one exists must fail, not silently - # backport nowhere. - printf '%s\n' "$block" | script_lines | grep -qF 'core.setFailed' -} - -@test "release lines sort numerically descending so 1.10 outranks 1.9" { - block="$(job_block prepare "$BACKPORT")" - [ -n "$block" ] - - # Numeric descending on major then minor is the ONLY reason release-1.10 - # ranks above release-1.9. A slip to the default lexicographic sort compares - # the names as strings, puts "release-1.9" first, and aims every backport at - # a stale line — with nothing going red. - printf '%s\n' "$block" | script_lines | grep -qF '.sort((a, b) => (b.maj - a.maj) || (b.min - a.min))' - - # The comparands must be numbers, not the captured strings. - printf '%s\n' "$block" | script_lines | grep -qF 'maj: Number(m[1]), min: Number(m[2])' - - # And the head of that sort is what `backport` uses. - printf '%s\n' "$block" | script_lines | grep -qF 'lines[0].name' -} - -# ── skip a target whose backport already merged ───────────────────────────── -# A second label event on a PR whose first backport already merged (e.g. -# backport-previous added after backport merged) re-enters this job for the -# already-delivered target. The cherry-pick is then empty, which the action's -# draft_commit_conflicts handling misreads as a real conflict and reports as a -# failed backport on the original PR — this guard skips the re-run instead. - -@test "the guard checks merged, not just closed, PRs on the deterministic branch" { - block="$(job_block backport "$BACKPORT")" - [ -n "$block" ] - - # The branch name the action itself would have used. Checking anything else - # answers a different question than "did THIS backport already land". - printf '%s\n' "$block" | script_lines | grep -qF 'backport-${context.payload.pull_request.number}-to-${targetBranch}' - - # Closed alone is not enough: a closed-without-merge PR is a stale or - # abandoned attempt, not proof the change reached the target. Loosening this - # to any closed PR would skip a real retry after a conflict was closed - # unmerged. - printf '%s\n' "$block" | script_lines | grep -qF "state: 'closed'" - printf '%s\n' "$block" | script_lines | grep -qF 'p.merged_at !== null' -} - -@test "both the checkout and the backport-action step are gated on the guard" { - block="$(job_block backport "$BACKPORT")" - [ -n "$block" ] - - # Losing the gate on either step still runs the action against a target that - # already has the change, reproducing the false comment the guard exists to - # prevent. - count="$(printf '%s\n' "$block" | code_lines | grep -cF "steps.guard.outputs.already_merged != 'true'" || true)" - [ "${count:-0}" -eq 2 ] -} - -@test "the guard runs before the checkout and the backport-action step" { - guard="$(step_line 'Check for existing merged backport' "$BACKPORT")" - checkout="$(step_line 'Checkout repository' "$BACKPORT")" - create="$(step_line 'Create back‑port PR' "$BACKPORT")" - [ -n "$guard" ] && [ -n "$checkout" ] && [ -n "$create" ] - - [ "$guard" -lt "$checkout" ] - [ "$guard" -lt "$create" ] -} - -# Mirrors release-changelog-behaviour.bats's convention for a github-script -# snippet: extract the decision expression verbatim in shape and execute it -# under node, because a grep for "merged_at" cannot tell an `.some()` from an -# `.every()` or a flipped comparison. Two cases are the ones that matter and -# are NOT interchangeable: the first-ever run (branch never existed, so the -# API returns an empty list) and a real retry (someone closed a conflicting -# backport PR without merging it). Both must NOT skip, or either the very -# first backport attempt or a legitimate re-creation after a manual close -# never runs. -@test "the guard's merged decision is correct on empty, closed-unmerged and closed-merged lists" { - command -v node >/dev/null || { echo "node unavailable; skipping"; return 0; } - tmp="$(mktemp -d)" - - cat > "$tmp/guard.js" <<'JS' -function alreadyMerged(prs) { - return prs.some(p => p.merged_at !== null); -} -const cases = { - 'first-run-empty': [], - 'closed-unmerged': [{ merged_at: null }], - 'closed-merged': [{ merged_at: '2026-01-01T00:00:00Z' }], -}; -const lines = Object.entries(cases).map(([name, prs]) => name + '=' + alreadyMerged(prs)); -require('fs').writeFileSync(1, lines.join('\n') + '\n'); -JS - - node "$tmp/guard.js" > "$tmp/out.txt" 2>"$tmp/err.txt" || { - echo "node failed:" >&2; cat "$tmp/err.txt" >&2; exit 1; } - - want="first-run-empty=false -closed-unmerged=false -closed-merged=true" - got="$(cat "$tmp/out.txt")" - [ "$got" = "$want" ] || { - echo "guard decision table is wrong." >&2 - echo "want:" >&2; printf '%s\n' "$want" | sed 's/^/ /' >&2 - echo "got:" >&2; printf '%s\n' "$got" | sed 's/^/ /' >&2 - exit 1 - } - - # The mirror must match the real guard's exact decision expression, not just - # the presence of "merged_at" somewhere in the step. Wrapped like the other - # tests in this file's final assertion: a bare failing command here, under a - # trap-installed EXIT handler, has been observed to abort the whole bats run - # instead of just this test on macOS. - block="$(job_block backport "$BACKPORT")" - [ -n "$block" ] - printf '%s\n' "$block" | script_lines | grep -qF 'prs.some(p => p.merged_at !== null)' || { - echo "backport.yaml's guard no longer uses prs.some(p => p.merged_at !== null)." >&2; exit 1; } - - rm -rf "$tmp" -} - -# ── backport concurrency ───────────────────────────────────────────────────── -# A run joins its concurrency group before `prepare`'s guard is evaluated, so -# nothing in that guard can keep a label event from disturbing the run already -# backporting the PR. The group key and `cancel-in-progress` close different -# halves of that and are pinned separately: either one alone still leaves a -# live failure, and neither reads load-bearing on its own. - -@test "the trigger surface the concurrency key is built for does not widen" { - # Both halves of the key are written against one trigger delivering `closed` or - # `labeled`, and neither degrades safely if anything else arrives. The group key - # routes only `labeled` aside, so a new event lands in the main group; - # `cancel-in-progress` is positive, so it does not cancel there; and `prepare` - # does not qualify it. The result occupies the single pending slot, evicts a - # genuine backport request, and delivers nothing. - # - # Pinned as the whole surface rather than just the types list, because a SIBLING - # trigger reaches the same harm without touching `types:` at all: anything - # carrying a `pull_request` payload resolves the PR number and so lands in the - # same per-PR group. `pull_request_review` is the realistic one. - # - # Deliberately strict: this also fails on a widening that would be harmless, - # `workflow_dispatch` for instance, whose payload carries no `pull_request` so - # the key degenerates to a shared group holding no genuine request. Separating - # the harmless case needs a list of which triggers carry that payload, and that - # list drifts. A red here means read the block above, not that the pin is wrong. - triggers="$(awk '/^on:/{inside=1;next} /^[a-z]/{inside=0} inside && /^ [a-z_]+:/{print $1}' "$BACKPORT")" - [ -n "$triggers" ] - [ "$(printf '%s\n' "$triggers" | wc -l | tr -d ' ')" -eq 1 ] - printf '%s\n' "$triggers" | grep -qF 'pull_request_target:' - - printf '%s\n' "$(code_lines < "$BACKPORT")" | grep -qF 'types: [closed, labeled]' -} - -@test "a label event that requests no backport gets its own concurrency group" { - line="$(code_lines < "$BACKPORT" | grep '^ group: backport-')" - [ -n "$line" ] - - # The per-PR discriminator, first: without it every PR's backport shares one - # group, and since a merge still cancels in that group, merging one PR kills - # another's in-flight backport. The assertions below all describe how a label - # event is separated from a merge WITHIN one PR, and every one of them stays - # green with the PR number deleted, so the widest failure of the three is the - # one nothing else here would notice. - printf '%s\n' "$line" | grep -qF 'backport-${{ github.workflow }}-${{ github.event.pull_request.number }}${{' - - count="$(printf '%s\n' "$line" | grep -o "github\.event\.action == 'labeled'" | wc -l | tr -d ' ')" - [ "${count:-0}" -eq 1 ] - - # The split is on the label NAMES, not on the action alone. Routing EVERY label - # event aside would move `backport` and `backport-previous` out of the group - # holding the run they have to queue behind, and one request could then evict - # the other. - # - # Pinned as the literal names, not as a count of `label.name != '`. That count - # measures the operator and says nothing about the operands, so renaming one - # name here and not in `prepare` — the half-finished rename — left it at two - # and stayed green, while a real `backport-previous` event took the `-label` - # branch and became free to cherry-pick alongside the merge run. - # Including the conjunction that joins the action check to the name checks. - # `&&` binds tighter than `||` in GitHub expressions, so flipping this one - # operator reads as `action == 'labeled' || (name != … && name != …)`, and on a - # `closed` event `github.event.label` is absent, both name checks are true, and - # EVERY event takes the suffix. A constant suffix is exactly as vacuous as no - # split at all. Every operand here was pinned before this line existed; the - # operator joining them was not. - # - # TRANSITIONAL: four names, not the original pair, because backport.yaml - # accepts both spellings while the org-level dosubot still applies the legacy - # ones. The namespaced names are genuine backport requests, so they belong on - # this side of the split exactly as the legacy ones do — routing them aside - # would hand a `kind/backport` event its own group and let it cherry-pick - # alongside the merge run, which is the failure the pin above describes. When - # dosubot's PR labelling is switched off and backport.yaml drops its legacy - # arms, this literal and the count below go back to the pair. - printf '%s\n' "$line" | grep -qF "github.event.action == 'labeled' && github.event.label.name != 'backport' && github.event.label.name != 'backport-previous' && github.event.label.name != 'kind/backport' && github.event.label.name != 'kind/backport-previous'" - - # And exactly four of them. The literal above is a substring check, so appending - # a fifth exclusion satisfies it unchanged — and a fifth exclusion is not - # cosmetic: the named label stops being routed aside, lands in the main group, - # takes its single pending slot, skips in `prepare`, and evicts a genuine - # backport request while delivering nothing. Same harm as widening `types:`. - # Presence and count answer different questions; this test needs both. - count="$(printf '%s\n' "$line" | grep -o "github\.event\.label\.name != '" | wc -l | tr -d ' ')" - [ "${count:-0}" -eq 4 ] - - # The suffix the condition selects, not only the condition. Asserting the - # operands alone leaves the whole split deletable — collapsing the tail to - # `&& '' || ''` puts every label back in the main group with the condition - # still reading correctly above it. Inverting it to `&& '' || '-label'` is - # worse than deleting it, because that moves the two backport requests aside - # and leaves the irrelevant labels sharing the group with the run in flight, - # which is the arrangement this file exists to prevent. Both survive every - # other assertion here. - printf '%s\n' "$line" | grep -qF "&& '-label' || ''" -} - -@test "a labeled event queues behind the backport in flight instead of cancelling it" { - # Labels set with the default GITHUB_TOKEN start no run, but labels set by - # third-party GitHub Apps do, and they arrive in bursts. Under a plain `true` - # each one killed the backport the merge had just started, leaving a cancelled - # run next to a green one and the work repeated by whichever label came last. - # With `prepare` now narrowed, no burst label requalifies to repeat it, - # so dropping this arm does not restore the old noisy delivery — it loses the - # backport. `cancel-in-progress: true` is the obvious thing for the next reader - # to normalise this back to, so pin its absence too. - count="$(code_lines < "$BACKPORT" | grep -cF " cancel-in-progress: \${{ github.event.action == 'closed' }}" || true)" - [ "${count:-0}" -eq 1 ] - - # Positively, not as `!= 'labeled'`. The two are equivalent only while `types:` - # holds exactly `closed` and `labeled`; the negative form hands cancellation to - # any third type added later, in the main group, which is the behaviour this - # arm removes. - count="$(code_lines < "$BACKPORT" | grep -c "cancel-in-progress:.*!= 'labeled'" || true)" - [ "${count:-0}" -eq 0 ] - - count="$(code_lines < "$BACKPORT" | grep -c '^ cancel-in-progress: true$' || true)" - [ "${count:-0}" -eq 0 ] -} - -@test "both backport jobs bound how long a queued request can wait" { - # These ceilings exist because of the queuing above, not for their own sake. A - # cancelling key disposed of a stuck run by killing it; queuing makes the next - # genuine request wait behind it instead, and an unbounded job on the 6-hour - # default turns one wedged job into a six-hour hole in the release line. - # Nothing else in this file makes them look load-bearing, so a tidy-up that - # drops them reads as harmless. - # - # Scope, since the test name says "how long a queued request can wait": what is - # bounded here is a stuck JOB. A run can wedge at the RUN level with every job - # already finished, and `timeout-minutes` is job-level with no run-level - # equivalent, so that state holds the group until the run is cancelled. These - # ceilings do not reach it and are not meant to. - # Asserted as a bound, not as presence. The hazard is the SIZE of the wait, so - # a pin that only checks the key exists is satisfied by `timeout-minutes: 355`, - # which restores the hole to within five minutes of the default it was written - # against. - # - # Bounded on BOTH sides, because both directions break it. Too high restores - # the hole; too low cancels a real backport that was only waiting for a runner, - # and an upper-bound-only assertion stays green at `timeout-minutes: 1`. - # - # The floor is judgement, not measurement, and says so. `timeout-minutes` - # bounds execution, and every execution measured on this workflow has been - # seconds, so no observation argues for any particular floor. No maximum is - # quoted deliberately: a superlative over a growing set goes stale on the next - # slower run. What the floor guards is a ceiling set low enough to cancel a - # cherry-pick much larger than anything seen yet; ten minutes is far above - # every leg measured. Per JOB, not per run, and not over the runner queue, - # which no `timeout-minutes` reaches. - for job in prepare backport; do - block="$(job_block "$job" "$BACKPORT" | code_lines)" - [ -n "$block" ] - minutes="$(printf '%s\n' "$block" | grep -o '^ timeout-minutes: [0-9][0-9]*' | grep -o '[0-9][0-9]*$')" - [ -n "$minutes" ] - [ "$minutes" -ge 10 ] - [ "$minutes" -le 30 ] - done -} - -@test "each backport trigger reads only the labels it is entitled to" { - block="$(job_block prepare "$BACKPORT" | code_lines)" - [ -n "$block" ] - - # The cumulative label set answers for the merge and for nothing else. Left - # ungated it re-enters this job on every later unrelated label — an automated - # size/* or kind/* — for a merged PR still carrying `backport`, redoing a - # backport that has already been delivered. - # - # TRANSITIONAL: four reads, not the original pair. backport.yaml accepts the - # namespaced and the legacy spelling of each label while the org-level dosubot - # still applies the legacy ones; when its PR labelling is switched off and the - # legacy arms are dropped, this literal and the count below go back to two. - printf '%s\n' "$block" | grep -qF "(github.event.action == 'closed' && (contains(github.event.pull_request.labels.*.name, 'backport') || contains(github.event.pull_request.labels.*.name, 'backport-previous') || contains(github.event.pull_request.labels.*.name, 'kind/backport') || contains(github.event.pull_request.labels.*.name, 'kind/backport-previous')))" - - # And the guard reads that set exactly four times, all of them inside the gated - # disjunct above, so it cannot be joined by an ungated one that quietly - # restores the old behaviour while the assertion above stays green. - # - # Counted as OCCURRENCES, not lines. `grep -c` counts matching lines, and all - # four reads live on one line here, so it answers 1 and keeps answering 1 when a - # fifth read is appended to that same line — which is exactly the restoration - # this is meant to catch. Only an append on a NEW line would have moved it. - count="$(printf '%s\n' "$block" | grep -o 'contains(github\.event\.pull_request\.labels' | wc -l | tr -d ' ')" - [ "${count:-0}" -eq 4 ] - - # A label event answers for the label it carries. Both spellings of each name, - # for the transitional reason given above. - printf '%s\n' "$block" | grep -qF "(github.event.action == 'labeled' && (github.event.label.name == 'backport' || github.event.label.name == 'backport-previous' || github.event.label.name == 'kind/backport' || github.event.label.name == 'kind/backport-previous'))" - - # The two conjuncts the label logic hangs off, which the label assertions above - # would not miss. `base.ref == 'main'` is what stops the bot backporting its own - # output: its PRs target release-X.Y, so they cannot satisfy it however often a - # labeler re-applies `backport` to a `[Backport release-X.Y]` title. docs/release.md - # calls that architectural protection, which is only true while the line is here. - # With their trailing conjunctions, for the reason the group-key test spells - # out: flipping either `&&` to `||` turns the guard into a disjunction that - # fires on every merge to main and on labels applied to open PRs. That one does - # not produce a wrong backport, because the `backport` job re-checks both - # independently, but it runs the job when nothing asked for it. - # In BOTH jobs, because docs/release.md rests the architectural protection on - # both `if:` blocks. Losing either copy alone is survivable (the other still - # skips, and `backport` needs `prepare`), so this pins a duplicate rather than - # a hole — but an untested duplicate is how a duplicate stops being one. - for job in prepare backport; do - b="$(job_block "$job" "$BACKPORT" | code_lines)" - [ -n "$b" ] - printf '%s\n' "$b" | grep -qF "github.event.pull_request.base.ref == 'main' &&" - printf '%s\n' "$b" | grep -qF 'github.event.pull_request.merged == true &&' - done -} - -@test "the audit accepts exactly the backport labels the bot acts on" { - # cmd/backport-audit answers "has everything labelled for this line landed on - # it" and exits non-zero while anything has not, so docs/release.md gates a - # patch cut on it. It reaches its candidates with `gh pr list --label`, which - # matches a label by exact name. - # - # Pin the two sets equal rather than pinning either one's contents: the - # audit must not miss a spelling the bot honours, and must not keep one the - # bot has retired. Both directions matter, because the legacy names are - # transitional and the two files have to drop them together. - [ -f "$AUDIT" ] - - # Every name the bot tests for, from the label-detection script. - bot="$(grep -o "labels\.includes('[^']*')" "$BACKPORT" | sed "s/.*('//;s/')//" | sort -u)" - [ -n "$bot" ] - - # Every backport label spelled as a Go string literal in the audit. Its own - # prose quotes these names with backticks, not double quotes, so a comment - # cannot satisfy this. - audit="$(grep -o '"\(kind/\)\{0,1\}backport\(-previous\)\{0,1\}"' "$AUDIT" | tr -d '"' | sort -u)" - [ -n "$audit" ] - - [ "$bot" = "$audit" ] - - # Both requests are represented, so the sets cannot agree by both having lost - # the same half. - printf '%s\n' "$audit" | grep -qx 'kind/backport' - printf '%s\n' "$audit" | grep -qx 'kind/backport-previous' -} diff --git a/hack/sign-release-assets.bats b/hack/sign-release-assets.bats index 9cfd25aca..0dfa06763 100644 --- a/hack/sign-release-assets.bats +++ b/hack/sign-release-assets.bats @@ -161,13 +161,13 @@ cleanup() { cleanup } -@test "docs/release.md tells users to verify the identity the script verifies" { +@test "docs/release/signatures-and-sbom.md tells users to verify the identity the script verifies" { regexp="$(sed -n "s/^IDENTITY_REGEXP='\(.*\)'\$/\1/p" "$SCRIPT")" if [ -z "$regexp" ]; then echo "IDENTITY_REGEXP not found in $SCRIPT" >&2 exit 1 fi - grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release.md" + grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release/signatures-and-sbom.md" } @test "fails when the signature does not verify" { diff --git a/hack/sign-release-assets.sh b/hack/sign-release-assets.sh index e8120feed..6549ee4f9 100755 --- a/hack/sign-release-assets.sh +++ b/hack/sign-release-assets.sh @@ -26,8 +26,9 @@ # copies along with its other assets, and a stable release gets its own. # # The bundle is verified before the script returns, with the same certificate -# identity users are told to check in docs/release.md, so a release whose -# signature that check would reject fails here instead of being uploaded. +# identity users are told to check in docs/release/signatures-and-sbom.md, so a +# release whose signature that check would reject fails here instead of being +# uploaded. # # Needs syft and cosign on PATH, and `id-token: write` in the job. set -euo pipefail @@ -39,7 +40,7 @@ BUNDLE=cozystack-checksums.txt.sigstore.json # The workflows that publish releases, at the refs they run from: tags.yaml for a # pre-release, on a v* tag push, and promote-rc.yaml for a stable one, dispatched # from main only. A run from any other ref signs, but fails the check below. -# docs/release.md shows users the same pattern. +# docs/release/signatures-and-sbom.md shows users the same pattern. IDENTITY_REGEXP='^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)' OIDC_ISSUER=https://token.actions.githubusercontent.com diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index ef91dc177..f0c597583 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -91,6 +91,76 @@ func validateAltinityApplicationRef(ref corev1.TypedLocalObjectReference) error return nil } +// altinityCredentialsPropagation bounds how long kubelet takes to rewrite +// the mounted cozy-backups-creds in the ClickHouse Pod after it changes: +// its sync period (a minute by default) plus the Secret cache. About 75s was +// measured; the margin covers the rest. +const altinityCredentialsPropagation = 2 * time.Minute + +// credentialsPropagationWait returns how long to hold back creating the +// strategy Job named jobName in namespace, so that the clickhouse-backup +// sidecar, which reads the projected credentials from a mounted file, sees +// their latest content. A Job that already exists, or a namespace without +// projected credentials, does not wait. +func credentialsPropagationWait(ctx context.Context, c client.Client, secretName, namespace, jobName string) (time.Duration, error) { + if secretName == "" { + return 0, nil + } + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: jobName}, &batchv1.Job{}); err == nil { + return 0, nil + } else if !apierrors.IsNotFound(err) { + return 0, err + } + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secretName}, secret); err != nil { + if apierrors.IsNotFound(err) { + return 0, nil + } + return 0, err + } + raw := secret.Annotations[credentialsUpdatedAtAnnotation] + if raw == "" { + return 0, nil + } + // The projector rewrites the stamp only when the content changes, so a + // stamp this hold cannot use stays as it is: holding on it would block + // every Job in the namespace for good. + updatedAt, err := time.Parse(time.RFC3339, raw) + if err != nil { + getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation) + return 0, nil + } + if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) { + getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw) + return 0, nil + } + return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil +} + +// clearCredentialsWait drops the Ready condition credentialsWaitCondition +// left behind, once the job no longer waits. It reports whether it did. +func clearCredentialsWait(conditions *[]metav1.Condition) bool { + ready := apimeta.FindStatusCondition(*conditions, "Ready") + if ready == nil || ready.Reason != credentialsWaitReason { + return false + } + return apimeta.RemoveStatusCondition(conditions, "Ready") +} + +const credentialsWaitReason = "WaitingForCredentials" + +// credentialsWaitCondition is the Ready condition of a job held back by +// credentialsPropagationWait. +func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition { + return metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: credentialsWaitReason, + Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar", + wait.Round(time.Second), secretName), + } +} + // jobNameForBackupJob returns the deterministic batch/v1.Job name used for // the backup pod that drives a BackupJob. Idempotency relies on this being // pure: a retried reconcile must hit the same Job name. @@ -293,6 +363,18 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, j.Namespace, jobNameForBackupJob(j)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&j.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityJob(ctx, j, j.Namespace, jobNameForBackupJob(j), altinityModeBackup, map[string]string{ @@ -343,6 +425,11 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return r.markBackupJobFailed(ctx, j, message) default: + if clearCredentialsWait(&j.Status.Conditions) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } @@ -712,6 +799,18 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Altinity strategy: %v", err)) } + wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, targetNamespace, jobNameForRestoreJob(restoreJob)) + if err != nil { + return ctrl.Result{}, err + } + if wait > 0 { + apimeta.SetStatusCondition(&restoreJob.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait)) + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: wait}, nil + } + batchJob, err := r.ensureAltinityRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob), altinityModeRestore, map[string]string{ @@ -748,6 +847,11 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res return r.markRestoreJobFailed(ctx, restoreJob, message) default: + if clearCredentialsWait(&restoreJob.Status.Conditions) { + if err := r.Status().Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: altinityPollInterval}, nil } } diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 7f2c906aa..49da21fa0 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -3,12 +3,14 @@ package backupcontroller import ( "context" + "errors" "strings" "testing" "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -20,6 +22,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -1169,3 +1172,539 @@ func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) { }) } } + +func projectedCredentials(namespace string, updatedAt time.Time) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: updatedAt.UTC().Format(time.RFC3339)}, + }, + } +} + +// The ClickHouse sidecar reads the projected credentials from a mounted +// file that kubelet rewrites only some time after the Secret changes, so a +// strategy Job started right after a projection would reach a sidecar with +// no or stale credentials. +func TestReconcileAltinity_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + cases := []struct { + name string + secret *corev1.Secret + jobExists bool + wantWait bool + }{ + {name: "credentials changed just now", secret: projectedCredentials("tenant-test", time.Now()), wantWait: true}, + {name: "credentials changed long ago", secret: projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))}, + {name: "no projected credentials", secret: nil}, + {name: "the Job already runs", secret: projectedCredentials("tenant-test", time.Now()), jobExists: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + objs := []client.Object{backupJob, newAltinityStrategy("clickhouse-strategy")} + if tc.secret != nil { + objs = append(objs, tc.secret) + } + if tc.jobExists { + objs = append(objs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}}) + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), clientfake.NewClientBuilder().WithObjects(objs...)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, err := r.reconcileAltinity(ctx, j, resolved) + if err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + ready := meta.FindStatusCondition(j.Status.Conditions, "Ready") + if tc.wantWait { + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created %d time(s) before the credentials could reach the sidecar", len(jobs.Items)) + } + if res.RequeueAfter <= 0 || res.RequeueAfter > altinityCredentialsPropagation { + t.Errorf("RequeueAfter = %v, want within (0, %v]", res.RequeueAfter, altinityCredentialsPropagation) + } + if ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } + return + } + if len(jobs.Items) != 1 { + t.Errorf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + if ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("Ready = %+v, the job should not wait", ready) + } + }) + } +} + +func TestReconcileAltinityRestore_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now()))) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + res, err := rr.reconcileAltinityRestore(ctx, rj, backup) + if err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created before the credentials could reach the sidecar") + } + if res.RequeueAfter <= 0 { + t.Errorf("RequeueAfter = %v, want a wait", res.RequeueAfter) + } + if ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCredentials" { + t.Errorf("Ready = %+v, want WaitingForCredentials", ready) + } +} + +// Without credentials projection configured there is no Secret to wait +// for; client-go refuses a Get with an empty name, so it must not be asked. +func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy")). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } +} + +func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cozy-backups-creds", + Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp}, + }, + } +} + +func waitingForCredentials() []metav1.Condition { + return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)} +} + +// Once the hold is over and the Job runs, a Ready condition still saying the +// job is waiting for credentials describes a state the job has left. +func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()}, + } + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.BackupJob{})) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("reconcileAltinity: %v", err) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items)) + } + stored := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + +func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning, + Conditions: waitingForCredentials()}, + } + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithStatusSubresource(&backupsv1alpha1.RestoreJob{})) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + stored := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" { + t.Errorf("stored Ready = %+v while the Job runs", ready) + } +} + +// The stamp is only ever rewritten when the projected content changes, so a +// value the hold cannot use must not hold the job: an unreadable one would +// fail every reconcile, and one far in the future would wait again on each. +// A stamp a little ahead, as a skewed clock writes it, still waits, but only +// until that stamp plus the propagation time. +func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) { + cases := []struct { + name string + stamp string + wantMin time.Duration + wantMax time.Duration + }{ + {name: "unreadable", stamp: "garbage"}, + {name: "far in the future", stamp: "3000-01-01T00:00:00Z"}, + {name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339), + wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build() + wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job") + if err != nil { + t.Fatalf("credentialsPropagationWait: %v", err) + } + if wait < tc.wantMin || wait > tc.wantMax { + t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax) + } + }) + } +} + +func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) { + conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}} + if clearCredentialsWait(&conditions) { + t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady") + } + if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" { + t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready) + } +} + +// A read the hold cannot complete must not release the Job: starting it +// before the credentials reach the sidecar is the failure the hold prevents. +func TestReconcileAltinity_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"), + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err == nil { + t.Errorf("reconcileAltinity returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a strategy Job was created although the %s could not be read", tc.name) + } + }) + } +} + +func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testing.T) { + cases := []struct { + name string + fail func(key client.ObjectKey, obj client.Object) bool + }{ + {name: "credentials Secret", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isSecret := obj.(*corev1.Secret) + return isSecret && key.Name == "cozy-backups-creds" + }}, + {name: "strategy Job", fail: func(key client.ObjectKey, obj client.Object) bool { + _, isJob := obj.(*batchv1.Job) + return isJob + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: now, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + builder := clientfake.NewClientBuilder(). + WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob, + projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if tc.fail(key, obj) { + return apierrors.NewInternalError(errors.New("etcd unavailable")) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder) + rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + ctx := context.Background() + rj := &backupsv1alpha1.RestoreJob{} + if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err == nil { + t.Errorf("reconcileAltinityRestore returned no error although the %s could not be read", tc.name) + } + jobs := &batchv1.JobList{} + if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Errorf("a restore Job was created although the %s could not be read", tc.name) + } + }) + } +} + +// A backup held for the credentials keeps the archive name it was given from +// its persisted StartedAt: the held reconciles write the job's status, and the +// Pod created after the hold, and the Backup recorded after it, must still +// name the same archive, which is stamped no later than the Backup's takenAt. +func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + secret := projectedCredentials("tenant-test", time.Now()) + r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), + clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityArchiveStrategy("clickhouse-strategy"), secret)) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + ctx := context.Background() + const want = "clickhouse-ch-test-20260304T050607Z" + reconcile := func(step string) { + t.Helper() + j := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil { + t.Fatalf("%s: get BackupJob: %v", step, err) + } + if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil { + t.Fatalf("%s: reconcileAltinity: %v", step, err) + } + } + + reconcile("held") + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); !apierrors.IsNotFound(err) { + t.Fatalf("the Job exists while the credentials wait should hold it: %v", err) + } + + if err := r.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + secret.Annotations[credentialsUpdatedAtAnnotation] = time.Now().Add(-altinityCredentialsPropagation - time.Second).UTC().Format(time.RFC3339) + if err := r.Update(ctx, secret); err != nil { + t.Fatalf("age the credentials stamp: %v", err) + } + reconcile("after the wait") + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want { + t.Fatalf("backup Pod was rendered with %q after the wait, want --archive=%s", got, want) + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + reconcile("completed") + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want { + t.Errorf("Backup records archive %q, want %q", got, want) + } + if stamp := created.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout); stamp < "20260304T050607Z" { + t.Errorf("Backup takenAt %s is earlier than its archive's stamp", stamp) + } +} diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 941b00323..6376c8e87 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -36,6 +36,7 @@ type s3ObjectInfo struct { type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) + StatObject(ctx context.Context, bucket, key string) (int64, error) } // s3Connection is what a driver resolves from its rendered strategy to @@ -53,6 +54,9 @@ type s3Connection struct { // what the writer of the backup trusted, no more. CAPEM []byte CAOnly bool + // InsecureSkipVerify mirrors an explicit admin opt-in a strategy + // already applies to its own uploads; it is never a fallback. + InsecureSkipVerify bool } // objectStore reads back what a backup wrote: the Secrets it was written @@ -104,14 +108,18 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { Region: conn.Region, Secure: conn.Secure, } - if conn.Secure && len(conn.CAPEM) > 0 { + if conn.Secure && (len(conn.CAPEM) > 0 || conn.InsecureSkipVerify) { // Start from minio's transport to keep its proxy and timeout settings. tr, err := minio.DefaultTransport(true) if err != nil { return nil, err } - if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { - return nil, err + if len(conn.CAPEM) > 0 { + if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { + return nil, err + } + } else { + tr.TLSClientConfig.InsecureSkipVerify = true } opts.Transport = tr } @@ -148,6 +156,18 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) StatObject(ctx context.Context, bucket, key string) (int64, error) { + info, err := l.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + if err != nil { + return 0, err + } + // minio-go reports -1 when the response carries no Content-Length. + if info.Size < 0 { + return 0, fmt.Errorf("object %s/%s: the endpoint reported no size", bucket, key) + } + return info.Size, nil +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go index 3d9c35b23..aff417344 100644 --- a/internal/backupcontroller/artifact_size_test.go +++ b/internal/backupcontroller/artifact_size_test.go @@ -14,6 +14,7 @@ import ( "math/big" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -37,6 +38,16 @@ type fakeS3Lister struct { bucket, prefix string recursive bool calls int + + statSize int64 + statErr error + statKey string +} + +func (f *fakeS3Lister) StatObject(_ context.Context, bucket, key string) (int64, error) { + f.calls++ + f.bucket, f.statKey = bucket, key + return f.statSize, f.statErr } func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { @@ -167,6 +178,11 @@ func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s return nil, ctx.Err() } +func (blockingLister) StatObject(ctx context.Context, _, _ string) (int64, error) { + <-ctx.Done() + return 0, ctx.Err() +} + // withShortProbeTimeout shortens the probe timeout and fails the test if fn // is not back well before the default timeout would have expired. func withShortProbeTimeout(t *testing.T, fn func()) { @@ -233,6 +249,76 @@ func TestFailBackupArtifact(t *testing.T) { }) } +// unrelatedCAPEM mints a CA that signed nothing an httptest server serves: +// every httptest server shares one certificate. +func unrelatedCAPEM(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +// TestMinioObjectLister_EnforcesTheCA pins that a configured CA is the +// trust the client verifies with, whether alone or on top of the system +// roots: an endpoint whose certificate it did not sign is refused. +func TestMinioObjectLister_EnforcesTheCA(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1") + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + })) + defer srv.Close() + // minio-go retries a TLS failure with backoff; a refusal needs no more + // than the first attempt. + stat := func(ca []byte, caOnly bool) error { + l, err := newMinioObjectLister(s3Connection{ + Endpoint: strings.TrimPrefix(srv.URL, "https://"), Secure: true, + AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1", + CAPEM: ca, CAOnly: caOnly, + }) + if err != nil { + t.Fatalf("newMinioObjectLister: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = l.StatObject(ctx, "cozy-backups", "k") + return err + } + signer := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + for _, caOnly := range []bool{true, false} { + if err := stat(signer, caOnly); err != nil { + t.Errorf("CAOnly=%v: the CA that signed the endpoint: %v", caOnly, err) + } + if err := stat(unrelatedCAPEM(t), caOnly); err == nil { + t.Errorf("CAOnly=%v: StatObject succeeded against a certificate the CA did not sign, want a TLS error", caOnly) + } + } +} + +// TestMinioObjectLister_StatWithoutSize pins that a HEAD answer without +// Content-Length, which minio-go turns into size -1, is not a size. +func TestMinioObjectLister_StatWithoutSize(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + })) + defer srv.Close() + l, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1"}) + if err != nil { + t.Fatalf("newMinioObjectLister: %v", err) + } + if size, err := l.StatObject(context.Background(), "cozy-backups", "k"); err == nil { + t.Errorf("StatObject = %d, nil; want an error for a response without Content-Length", size) + } +} + // TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system // roots" and "the CA alone". The system pool holds the endpoint's // certificate, and the configured CA is an unrelated one, so the endpoint @@ -243,18 +329,7 @@ func TestEndpointTrustPool_CAOnly(t *testing.T) { system := x509.NewCertPool() system.AddCert(endpoint.Certificate()) - // Every httptest server shares one certificate, so the unrelated CA is - // minted here. - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} - der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) - if err != nil { - t.Fatal(err) - } - unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + unrelatedCA := unrelatedCAPEM(t) verifies := func(caOnly bool) bool { pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone()) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 819efc10f..19f255c6c 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -175,13 +175,11 @@ const ( // against an unreachable target reproduces the same FATAL. cnpgRecoveryMaxInspectPods = 5 - // Cap on the wall-clock time a BackupJob can spend observing a - // cnpg.io/Backup stuck in phase=failed before the driver gives up and - // marks the BackupJob Failed. CNPG can transition out of "failed" on - // internal retries (e.g. transient instance-manager restart), so we - // don't fail on the first observation. But without this cap, a - // permanently-broken Backup would pin the BackupJob in Running forever, - // blocking later runs and wedging the Plan-controller queue. + // Cap on the wall-clock time a BackupJob waits for the barman-cloud + // plugin rollout before the cnpg.io/Backup exists. A Backup that exists + // is not capped: CloudNativePG keeps a healthy one pending while another + // backup of the same Cluster runs, or while it restarts one on another + // instance, so no fixed deadline tells those apart from a stuck one. cnpgDefaultBackupDeadline = 30 * time.Minute cnpgClusterHealthyPhase = "Cluster in healthy state" @@ -453,44 +451,46 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp return ctrl.Result{}, nil case cnpgBackupPhaseFailed: - // CNPG can transition the same Backup CR back out of "failed" when it - // retries internally (e.g., after a transient instance-manager - // restart), so we don't fail the BackupJob on the first observation. - // But without a cap a permanently-broken Backup would pin the - // BackupJob in Running forever; once StartedAt + cnpgDefaultBackupDeadline - // has elapsed and the cnpg.io/Backup is still failed, we give up. - if cnpgBackupDeadlineExceeded(j.Status.StartedAt) { - final := message - if final == "" { - final = fmt.Sprintf( - "cnpg.io Backup remained in phase=failed past %s deadline", - cnpgDefaultBackupDeadline) - } - return r.markBackupJobFailed(ctx, j, final) - } - if message == "" { - message = "cnpg.io Backup reported phase=failed; awaiting recovery or retry" - } - apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ - Type: "Ready", - Status: metav1.ConditionFalse, - Reason: "CNPGBackupTransientFailure", - Message: message, - }) - if err := r.Status().Update(ctx, j); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil + // CloudNativePG does not reconcile a Backup out of a terminal phase, + // so waiting on a failed one cannot change the outcome. + return r.markBackupJobFailed(ctx, j, cnpgBackupFailedMessage(cnpgBackup, message)) default: + waiting := fmt.Sprintf("waiting for cnpg.io/Backup %s/%s, phase %q", + cnpgBackup.Namespace, cnpgBackup.Name, phase) + if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "WaitingForCNPGBackup", + Message: waiting, + }) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } } +// cnpgBackupFailedMessage carries the error CloudNativePG recorded. When the +// plugin run itself failed that is only its exit status, and the storage +// error behind it is logged by the plugin-barman-cloud sidecar. +func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string { + if errText == "" { + errText = "no error recorded" + } + message := fmt.Sprintf("cnpg.io/Backup %s/%s failed: %s", b.Namespace, b.Name, errText) + if strings.Contains(errText, "exit status") { + message += "; the cause is in the log of the plugin-barman-cloud sidecar of the instance that ran it" + } + return message +} + // cnpgBackupDeadlineExceeded reports whether enough wall-clock time has -// elapsed since the BackupJob started that we should stop polling a stuck -// cnpg.io/Backup and fail the run. Returns false when StartedAt is nil so -// the very first reconcile (which sets StartedAt) does not trip the gate. +// elapsed since the BackupJob started that we should stop waiting for the +// barman-cloud plugin rollout and fail the run. Returns false when +// StartedAt is nil so the very first reconcile (which sets StartedAt) does +// not trip the gate. func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool { if startedAt == nil { return false diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 697843d9e..551209fac 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2332,6 +2332,154 @@ func TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion(t *testi } } +// CloudNativePG never reconciles a Backup out of a terminal phase (the +// backup controller returns early on failed and completed), so a failed +// Backup fails the BackupJob at once. Any other phase keeps the BackupJob +// waiting past the backup deadline: CNPG keeps a healthy Backup pending +// while another backup of the Cluster runs or while it restarts one. +func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + const ns, appName = "tenant", "pg" + clusterName := cnpgClusterNameForApp(appName) + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + + cases := []struct { + name string + phase string + errText string + startedAt time.Time + wantPhase backupsv1alpha1.BackupJobPhase + wantMessage []string + notInMessage []string + }{ + {name: "failed fails at once", phase: cnpgBackupPhaseFailed, errText: "rpc error: code = Unknown desc = exit status 1", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "exit status 1", "plugin-barman-cloud"}}, + {name: "failed without an error text fails at once", phase: cnpgBackupPhaseFailed, startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "no error recorded"}, notInMessage: []string{"plugin-barman-cloud"}}, + {name: "failed before the plugin ran points nowhere else", phase: cnpgBackupPhaseFailed, errText: "cannot backup a hibernated cluster", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "hibernated"}, notInMessage: []string{"plugin-barman-cloud"}}, + {name: "pending keeps waiting", phase: "pending", startedAt: time.Now(), + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, + {name: "pending is not capped by the backup deadline", phase: "pending", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}}, + {name: "no phase yet keeps waiting", phase: "", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg"}}, + {name: "started keeps running", phase: "started", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "started"}}, + {name: "walArchivingFailing keeps waiting", phase: "walArchivingFailing", startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "walArchivingFailing"}}, + {name: "completed succeeds", phase: cnpgBackupPhaseComplete, startedAt: expired, + wantPhase: backupsv1alpha1.BackupJobPhaseSucceeded}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + startedAt := metav1.NewTime(tc.startedAt) + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, + Spec: strategyv1alpha1.CNPGSpec{ + Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, + }, + }, + } + cluster := &cnpgtypes.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"}, + } + cnpgBackup := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: job.Name, + backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace, + }}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, + Status: cnpgtypes.BackupStatus{Phase: tc.phase, Error: tc.errText}, + } + c := newCNPGStrategyTestClient(t, job, strategy, newPostgresApp(appName, ns), cluster, cnpgBackup) + r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)} + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}, + } + + j := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, err := r.reconcileCNPG(context.Background(), j, resolved) + if err != nil { + t.Fatalf("reconcileCNPG: %v", err) + } + if j.Status.Phase != tc.wantPhase { + t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) + } + + message := j.Status.Message + if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning { + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the cnpg.io/Backup is in progress, got %+v", res) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), persisted); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + ready := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForCNPGBackup" { + t.Fatalf("expected Ready=False/WaitingForCNPGBackup while waiting, got %+v", ready) + } + message = ready.Message + + // A pass that observes the same phase again must not write the + // BackupJob status: the driver polls every few seconds. + statusWrites := 0 + r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + statusWrites++ + return cl.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) + if _, err := r.reconcileCNPG(context.Background(), persisted, resolved); err != nil { + t.Fatalf("second reconcileCNPG: %v", err) + } + if statusWrites != 0 { + t.Errorf("an unchanged phase wrote the BackupJob status %d time(s)", statusWrites) + } + } + for _, want := range tc.wantMessage { + if !strings.Contains(message, want) { + t.Errorf("message %q does not mention %q", message, want) + } + } + for _, unwanted := range tc.notInMessage { + if strings.Contains(message, unwanted) { + t.Errorf("message %q mentions %q", message, unwanted) + } + } + + artifacts := &backupsv1alpha1.BackupList{} + if err := c.List(context.Background(), artifacts, client.InNamespace(ns)); err != nil { + t.Fatalf("list Backups: %v", err) + } + wantArtifacts := 0 + if tc.wantPhase == backupsv1alpha1.BackupJobPhaseSucceeded { + wantArtifacts = 1 + } + if len(artifacts.Items) != wantArtifacts { + t.Errorf("got %d Backup artifacts, want %d", len(artifacts.Items), wantArtifacts) + } + }) + } +} + // cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose // Cluster and instance pods a test swaps between passes, standing in for // CNPG rolling the instances. @@ -2516,8 +2664,8 @@ func TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup(t *testing.T) { if !apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { t.Errorf("%s condition = %+v, want True", backupCondPluginRolledOut, apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)) } - if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond != nil { - t.Errorf("Ready condition = %+v left over from the wait while the backup runs", cond) + if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready condition = %+v, want the wait on the cnpg.io/Backup rather than one left over from the rollout", cond) } } diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go index 4e0c3b75a..aaba89b3c 100644 --- a/internal/backupcontroller/credentials_projector.go +++ b/internal/backupcontroller/credentials_projector.go @@ -1,11 +1,14 @@ package backupcontroller import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "maps" "net/url" + "strconv" "strings" "time" @@ -15,6 +18,8 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" ) // CredentialsProjectionRequeue is the backoff between reconcile attempts @@ -73,9 +78,8 @@ func IsTransient(err error) bool { // BackupCredentialsConfig describes the platform-managed source Secret // produced by the cozy-backups Bucket, the per-tenant projection target // Secret name, and the bucket coordinates (endpoint used to format the -// FDB blob_credentials account-host key, region carried verbatim for -// drivers like clickhouse-backup that wire it through env, forcePathStyle -// projected as a flat key so the chart-emitted ClickHouse sidecar can read +// FDB blob_credentials account-host key, region carried verbatim, +// forcePathStyle projected so the chart-emitted ClickHouse sidecar can read // a platform value it cannot see directly). Values are wired from // environment variables in main() and the deployment chart populates them // from .Values.backupStorage. @@ -85,11 +89,11 @@ type BackupCredentialsConfig struct { TargetSecretName string Endpoint string Region string - // ForcePathStyle is the string "true"/"false" from - // backupStorage.forcePathStyle, projected into the credentials Secret - // under the forcePathStyle key. The app charts cannot read - // backupStorage.* directly, so the ClickHouse sidecar consumes it via - // an optional secretKeyRef as S3_FORCE_PATH_STYLE. + // ForcePathStyle is backupStorage.forcePathStyle as the chart + // normalises it ("true", "false", or "" when unset), projected into the credentials Secret + // under the forcePathStyle key and into clickhouse-backup.yml. The app + // charts cannot read backupStorage.* directly, so the ClickHouse + // sidecar takes it from there. ForcePathStyle string } @@ -107,10 +111,11 @@ func (c BackupCredentialsConfig) IsEnabled() bool { // the data so every default Strategy CR can consume the same Secret: // // - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — CNPG, MariaDB, Etcd. -// - accessKey / secretKey + bucketName / endpoint / region — ClickHouse -// sidecar. +// - accessKey / secretKey + bucketName / endpoint / region — Redis +// strategy, and ClickHouse sidecars rendered before clickhouse-backup.yml. // - cloud — Velero credentials file ([default] section). // - blob_credentials.json — FoundationDB backup_agent. +// - clickhouse-backup.yml — ClickHouse sidecar, mounted as its config file. // // The source Secret is read in two formats: // @@ -156,8 +161,8 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if creds.endpoint == "" { // Scheme-strip the env-var fallback to match the source-Secret path // (parseSourceSecret strips there). The projected `endpoint` key is - // consumed verbatim as S3_ENDPOINT by the ClickHouse sidecar, which - // expects a bare host:port — see https://cozystack.io/docs/v1/operations/services/backup-classes/. + // a bare host:port and each consumer adds its own scheme — see + // https://cozystack.io/docs/v1/operations/services/backup-classes/. creds.endpoint = stripScheme(cfg.Endpoint) } if creds.endpoint == "" { @@ -177,6 +182,20 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr } cloud := buildVeleroCredentialsFile(creds.accessKey, creds.secretKey) + clickHouseConfig, pathStyleErr, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle) + if pathStyleErr != nil { + // The chart refuses such a value at render time, so only an env set + // around it gets here. It concerns the ClickHouse file alone, and + // failing the projection would take every other driver's + // credentials down with it. + log.FromContext(ctx).Error(pathStyleErr, "left force_path_style out of "+clickHouseBackupConfigKey, "namespace", targetNamespace) + } + if err != nil { + return &ProjectionError{ + Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("build %s: %v", clickHouseBackupConfigKey, err), + } + } blob, err := buildFDBBlobCredentials(creds.endpoint, creds.accessKey, creds.secretKey) if err != nil { return &ProjectionError{ @@ -231,28 +250,36 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr if target.Data == nil { target.Data = map[string][]byte{} } + before := maps.Clone(target.Data) target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey) target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey) - // Mirror under the legacy bucket-controller key names so - // chart-emitted sidecars (clickhouse-backup) can consume the - // same Secret without a separate field map. + // Mirror under the legacy bucket-controller key names so the Redis + // strategy and older ClickHouse charts can consume the same Secret + // without a separate field map. target.Data["accessKey"] = []byte(creds.accessKey) target.Data["secretKey"] = []byte(creds.secretKey) target.Data["cloud"] = []byte(cloud) target.Data["blob_credentials.json"] = blob + target.Data[clickHouseBackupConfigKey] = clickHouseConfig // Always overwrite endpoint/bucket/region (delete if empty in // source) so a re-projection cannot leave stale values lingering - // after the bucket-controller rotates the source. Consumers - // (ClickHouse sidecar via secretKeyRef) must see exactly what the - // source carries, not a half-stale Secret. + // after the bucket-controller rotates the source. Consumers must + // see exactly what the source carries, not a half-stale Secret. setOrDelete(target.Data, "endpoint", creds.endpoint) setOrDelete(target.Data, "bucketName", creds.bucket) setOrDelete(target.Data, "region", creds.region) // forcePathStyle comes from the platform config (backupStorage. - // forcePathStyle), not the source Secret: the ClickHouse sidecar - // reads it as S3_FORCE_PATH_STYLE because the app chart cannot see + // forcePathStyle), not the source Secret: ClickHouse sidecars + // rendered before clickhouse-backup.yml read it as + // S3_FORCE_PATH_STYLE because the app chart cannot see // backupStorage.* directly. setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle) + if !maps.EqualFunc(before, target.Data, bytes.Equal) { + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + target.Annotations[credentialsUpdatedAtAnnotation] = time.Now().UTC().Format(time.RFC3339) + } return nil }); err != nil { // If the mutator refused to overwrite an unowned Secret, surface @@ -414,6 +441,48 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err return json.Marshal(doc) } +// credentialsUpdatedAtAnnotation records when the projected content last +// changed. A consumer that mounts the Secret as a volume sees a change only +// once kubelet rewrites the volume, so it uses this to tell how fresh its +// copy can be. +const credentialsUpdatedAtAnnotation = "backups.cozystack.io/credentials-updated-at" + +// clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup +// config. The ClickHouse chart mounts it as the sidecar's config file +// instead of passing the values as environment: clickhouse-backup reloads +// the file in each API handler that reaches S3, among them the +// create_remote, restore_remote and list calls the strategy makes, and +// kubelet rewrites a mounted Secret in place, so credentials projected +// after the pod started, or rotated since, reach a running sidecar. +const clickHouseBackupConfigKey = "clickhouse-backup.yml" + +// buildClickHouseBackupConfig renders that file. The projected endpoint is +// a bare host, and clickhouse-backup's S3 client rejects one without a +// scheme. A forcePathStyle that is not a boolean is left out of the file +// and reported in pathStyleErr; err is a failure to render the file at all. +func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) (config []byte, pathStyleErr, err error) { + s3 := map[string]any{ + "bucket": creds.bucket, + "endpoint": "https://" + creds.endpoint, + "access_key": creds.accessKey, + "secret_key": creds.secretKey, + } + // An empty region in the file would override clickhouse-backup's own + // us-east-1 default instead of leaving it in place. + if creds.region != "" { + s3["region"] = creds.region + } + if forcePathStyle != "" { + if pathStyle, perr := strconv.ParseBool(forcePathStyle); perr != nil { + pathStyleErr = fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle) + } else { + s3["force_path_style"] = pathStyle + } + } + config, err = yaml.Marshal(map[string]any{"s3": s3}) + return config, pathStyleErr, err +} + // stripScheme returns the endpoint stripped of any leading scheme prefix // (http:// or https://) and preserves the path portion if present. The // resulting form is what `host[:port][/path]` callers (MariaDB strategy, diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go index 2b4f0bb1c..74fd936d1 100644 --- a/internal/backupcontroller/credentials_projector_test.go +++ b/internal/backupcontroller/credentials_projector_test.go @@ -5,7 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "strings" "testing" + "time" + + "github.com/go-logr/logr/funcr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,6 +17,8 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" ) func newFakeClient(objs ...client.Object) client.Client { @@ -664,3 +670,188 @@ func TestProject_HappyPath_HTTPEndpoint(t *testing.T) { t.Errorf("blob account host not stripped: %+v", blob.Accounts) } } + +// The ClickHouse sidecar reads its S3 settings from this key as a mounted +// file, which kubelet refreshes in place: clickhouse-backup reloads its +// config file in each API handler that reaches S3, while its environment +// is fixed at start. +func TestProject_WritesClickHouseBackupConfig(t *testing.T) { + cases := []struct { + name string + forcePathStyle string + wantPathStyle *bool + }{ + {name: "path style on", forcePathStyle: "true", wantPathStyle: ptrBool(true)}, + {name: "path style off", forcePathStyle: "false", wantPathStyle: ptrBool(false)}, + {name: "path style unset", forcePathStyle: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.ForcePathStyle = tc.forcePathStyle + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v\n%s", clickHouseBackupConfigKey, err, got.Data[clickHouseBackupConfigKey]) + } + for key, want := range map[string]string{ + "bucket": "cozy-backups", + "endpoint": "https://seaweedfs-s3.tenant-root.svc:8333", + "region": "us-east-1", + "access_key": "AK", + "secret_key": "SK", + } { + if doc.S3[key] != want { + t.Errorf("s3.%s = %v, want %q", key, doc.S3[key], want) + } + } + pathStyle, set := doc.S3["force_path_style"] + switch { + case tc.wantPathStyle == nil && set: + t.Errorf("s3.force_path_style = %v, want it left to the clickhouse-backup default", pathStyle) + case tc.wantPathStyle != nil && pathStyle != *tc.wantPathStyle: + t.Errorf("s3.force_path_style = %v, want %v", pathStyle, *tc.wantPathStyle) + } + }) + } +} + +// clickhouse-backup defaults the S3 region to us-east-1; an empty value in +// its config file would replace that default rather than fall back to it. +func TestProject_ClickHouseBackupConfigOmitsAnEmptyRegion(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + ctx := context.Background() + cfg := defaultCfg() + cfg.Region = "" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if region, set := doc.S3["region"]; set { + t.Errorf("s3.region = %q, want it left to the clickhouse-backup default", region) + } +} + +// The parse only serves the ClickHouse file, while the same projection +// carries every other driver's credentials. An unparseable platform value +// must cost ClickHouse its path-style setting and say so, not fail the +// projection for all of them. +func TestProject_MalformedForcePathStyleSkipsOnlyTheClickHouseSetting(t *testing.T) { + c := newFakeClient(flatSourceSecret()) + var logged []string + ctx := log.IntoContext(context.Background(), funcr.New(func(prefix, args string) { + logged = append(logged, args) + }, funcr.Options{})) + cfg := defaultCfg() + cfg.ForcePathStyle = "yes" + if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil { + t.Fatalf("projection failed for every driver over a ClickHouse-only setting: %v", err) + } + got := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + if string(got.Data["AWS_ACCESS_KEY_ID"]) != "AK" { + t.Errorf("AWS_ACCESS_KEY_ID = %q, want the projected key", got.Data["AWS_ACCESS_KEY_ID"]) + } + if string(got.Data["forcePathStyle"]) != "yes" { + t.Errorf("forcePathStyle = %q, want the platform value carried verbatim", got.Data["forcePathStyle"]) + } + var doc struct { + S3 map[string]any `json:"s3"` + } + if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil { + t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err) + } + if doc.S3["access_key"] != "AK" { + t.Errorf("s3.access_key = %v, want the rest of the file written", doc.S3["access_key"]) + } + if pathStyle, set := doc.S3["force_path_style"]; set { + t.Errorf("s3.force_path_style = %v, want it left out", pathStyle) + } + found := false + for _, line := range logged { + if strings.Contains(line, `"error"`) && strings.Contains(line, `\"yes\"`) && strings.Contains(line, "force_path_style") { + found = true + } + } + if !found { + t.Errorf("no error naming the rejected value and the skipped setting was logged; got %q", logged) + } +} + +func ptrBool(b bool) *bool { return &b } + +// Consumers that read the Secret through a mounted volume need to know when +// its content last changed; an unchanged re-projection must keep the time. +func TestProject_StampsWhenContentChanges(t *testing.T) { + src := flatSourceSecret() + c := newFakeClient(src) + ctx := context.Background() + key := types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"} + stamp := func() string { + got := &corev1.Secret{} + if err := c.Get(ctx, key, got); err != nil { + t.Fatalf("projected Secret not found: %v", err) + } + return got.Annotations[credentialsUpdatedAtAnnotation] + } + + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("first projection: %v", err) + } + first := stamp() + at, err := time.Parse(time.RFC3339, first) + if err != nil || time.Since(at) > time.Minute { + t.Fatalf("first projection stamp %q is not a recent RFC3339 time: %v", first, err) + } + + old := &corev1.Secret{} + if err := c.Get(ctx, key, old); err != nil { + t.Fatalf("get projected Secret: %v", err) + } + old.Annotations[credentialsUpdatedAtAnnotation] = "2000-01-01T00:00:00Z" + if err := c.Update(ctx, old); err != nil { + t.Fatalf("age the stamp: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("unchanged re-projection: %v", err) + } + if got := stamp(); got != "2000-01-01T00:00:00Z" { + t.Fatalf("an unchanged re-projection moved the stamp to %q", got) + } + + rotated := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: src.Namespace, Name: src.Name}, rotated); err != nil { + t.Fatalf("get source Secret: %v", err) + } + rotated.Data["secretKey"] = []byte("SK2") + if err := c.Update(ctx, rotated); err != nil { + t.Fatalf("rotate source Secret: %v", err) + } + if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil { + t.Fatalf("rotated re-projection: %v", err) + } + if got := stamp(); got == "2000-01-01T00:00:00Z" { + t.Fatalf("a rotation did not move the stamp") + } +} diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 023682036..9aea96f31 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -2,10 +2,12 @@ package backupcontroller import ( "context" + "encoding/json" "errors" "fmt" "net/http" "sort" + "strings" "time" "github.com/go-logr/logr" @@ -49,15 +51,16 @@ var defaultObjectsMissing = prometheus.NewGaugeVec( ) // defaultObjectsForceReconciles counts the forced Helm upgrades the gate -// issued to materialise those objects. A counter that keeps climbing means -// the forced render is not producing the objects (e.g. a CRD is missing), +// issued to materialise those objects or to refresh their S3 endpoint. A +// counter that keeps climbing means the forced render is not producing the +// objects (e.g. a CRD is missing) or not changing their endpoint, // which is a different failure than the install-time race this gate closes. // A suspended release is NOT counted here: it is skipped before the patch, // so a paused release cannot masquerade as a render that keeps failing. var defaultObjectsForceReconciles = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cozystack_backup_default_objects_force_reconciles_total", - Help: "Number of forced HelmRelease reconciles issued to create missing platform backup objects.", + Help: "Number of forced HelmRelease reconciles issued to create missing platform backup objects or refresh their S3 endpoint.", }, []string{"namespace", "name"}, ) @@ -91,6 +94,11 @@ const ( // same lookup-gated template as the Strategy CRs, so it belongs to the // same existence check. defaultBackupStorageLocationName = "cozy-default" + + // endpointAnnotation carries the S3 endpoint the chart rendered into the + // object. The endpoint itself sits at a different path in every kind, + // and in some only as a bare host, so the gate compares this instead. + endpointAnnotation = "backups.cozystack.io/endpoint" ) var backupStorageLocationGVR = schema.GroupVersionResource{ @@ -118,8 +126,9 @@ var backupStorageLocationGVR = schema.GroupVersionResource{ // is resolvable — read from the same platform credentials Secret the // projector already consumes, so no new dependency and no `lookup` — it // checks that every object the platform BackupClass routes to actually -// exists, and forces one Helm upgrade when any is missing. A forced -// upgrade re-runs the lookups, so the gated templates materialise. +// exists, and forces one Helm upgrade when any is missing or carries a stale +// S3 endpoint. A forced upgrade re-runs the lookups, so the gated templates +// materialise with the current endpoint. // // It deliberately does not create the objects itself: their bodies are // values-driven (endpoint, region, forcePathStyle, the Altinity strategy's @@ -157,6 +166,14 @@ type DefaultObjectsGate struct { // then admin-managed and no release renders it, so there is nothing // to force. CredentialsHelmRelease types.NamespacedName + // CredentialsCOSISecretName is the Secret the COSI sidecar writes for the + // platform bucket's BucketAccess, in Config.SourceNamespace, from which + // CredentialsHelmRelease renders the credentials Secret. The sidecar + // writes it once per BucketAccess, so after a publishing.host change a + // re-granted BucketAccess carries the new endpoint while the credentials + // Secret keeps the old one; the gate compares the two. Empty disables + // the comparison. + CredentialsCOSISecretName string // VeleroNamespace is where the cozy-default BackupStorageLocation // lives. Empty skips the BSL check (velero.bslEnabled=false). VeleroNamespace string @@ -280,38 +297,48 @@ func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger // Info, not Error: every branch here is either transient (the // bucket is still being provisioned) or already exposed as a // metric, and this runs every minute for the life of the cluster. - logger.Info("default backup objects check failed", "error", err.Error()) + logger.Info("default backup objects check failed", "error", err.Error(), + "missing", missing, "forced", forced) case len(missing) > 0 && forced: - // The release named here is whichever one renders what is missing: + // The releases named here are the ones that render what is missing: // the credentials Secret comes from the bucket's -system // release, everything else from this chart's own. - logger.Info("forced a Helm upgrade to create missing default backup objects", - "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing) + logger.Info("forced a Helm upgrade to create missing or stale default backup objects", + "helmReleases", g.forcedReleasesFor(missing), "missing", missing) case len(missing) > 0: - logger.Info("default backup objects still missing, no force issued on this tick", - "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing, + logger.Info("default backup objects still missing or stale, no force issued on this tick", + "helmReleases", g.forcedReleasesFor(missing), "missing", missing, "minForceInterval", g.MinForceInterval.String()) default: logger.V(1).Info("all default backup objects present") } } -// forcedReleaseFor reports which release renders the given missing set, for -// logging only. reconcileCredentials returns the credentials Secret as the -// sole missing object, and it is never mixed with the routed objects — -// resolving the bucket name is a precondition for checking those at all. -func (g *DefaultObjectsGate) forcedReleaseFor(missing []string) types.NamespacedName { +// forcedReleasesFor reports which releases render the given entries, for +// logging only: the credentials Secret comes from CredentialsHelmRelease, +// everything else from HelmRelease. +func (g *DefaultObjectsGate) forcedReleasesFor(missing []string) []string { credsKey := fmt.Sprintf("Secret/%s", g.Config.SourceSecretName) - if len(missing) == 1 && missing[0] == credsKey && g.CredentialsHelmRelease.Name != "" { - return g.CredentialsHelmRelease + var out []string + seen := map[string]bool{} + for _, m := range missing { + release := g.HelmRelease.String() + if (m == credsKey || strings.HasPrefix(m, credsKey+" ")) && g.CredentialsHelmRelease.Name != "" { + release = g.CredentialsHelmRelease.String() + } + if !seen[release] { + seen[release] = true + out = append(out, release) + } } - return g.HelmRelease + return out } // Check resolves the platform bucket name, verifies that every object the -// BackupClass routes to exists, and forces one Helm upgrade when any is -// missing. It returns the missing objects (formatted for logs) and whether -// a forced upgrade was issued on this call. +// BackupClass routes to exists and carries the current S3 endpoint, and +// forces one Helm upgrade when any is missing or stale. It returns those +// objects (formatted for logs) and whether a forced upgrade was issued on this +// call. // // While the source Secret is absent or carries no bucket name, the gate // cannot evaluate the routed objects — their templates gate on the same @@ -345,23 +372,110 @@ func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) return g.reconcileCredentials(ctx) } + // The credentials comparison needs only the two Secrets, so its force goes + // out before the object reads: an unreadable BackupClass or Strategy must + // not keep the stale endpoint in the Secret every driver reads. + var needed []string + var forced bool + staleCreds, credsErr := g.staleCredentials(ctx, src) + errs := []error{credsErr} + if credsErr != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + } + if staleCreds != "" { + _, credsForced, err := g.force(ctx, g.CredentialsHelmRelease, &g.lastCredentialsForce, []string{staleCreds}) + needed = append(needed, staleCreds) + forced = credsForced + errs = append(errs, err) + } + backupClass := &backupsv1alpha1.BackupClass{} if err := g.Client.Get(ctx, client.ObjectKey{Name: g.BackupClassName}, backupClass); err != nil { - defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() - return nil, false, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err) + if credsErr == nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + } + return needed, forced, errors.Join(append(errs, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err))...) } - missing, err := g.missingObjects(ctx, backupClass) + // With provisionBucket=false (no bucket release) the chart renders + // backupStorage.endpoint rather than the Secret's, so the two may + // legitimately differ and there is nothing to compare. + want := "" + if g.CredentialsHelmRelease.Name != "" { + want = renderedEndpointFor(src.Data["endpoint"]) + } + state, err := g.inspectObjects(ctx, backupClass, want) if err != nil { - defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() - return nil, false, err - } - defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) - if len(missing) == 0 { - return nil, false, nil + if credsErr == nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + } + return needed, forced, errors.Join(append(errs, err)...) } + defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(state.missing))) - return g.force(ctx, g.HelmRelease, &g.lastForce, missing) + if objects := append(state.missing, state.stale...); len(objects) > 0 { + sort.Strings(objects) + _, objectsForced, err := g.force(ctx, g.HelmRelease, &g.lastForce, objects) + needed = append(needed, objects...) + forced = forced || objectsForced + errs = append(errs, err) + } + return needed, forced, errors.Join(errs...) +} + +// staleCredentials reports the credentials Secret, formatted for logs, when +// its endpoint differs from the one in the COSI Secret it is rendered from, +// comparing against what packages/system/bucket/templates/user-credentials.yaml +// writes for it. The COSI Secret being absent (the BucketAccess is being +// re-granted) leaves nothing to compare against and is not staleness, and +// neither is a source Secret the bucket chart does not render: the COSI Secret +// itself (the projector accepts one) or one without the endpoint key. A +// BucketInfo without an endpoint is not a target either: rendering from it +// writes an empty endpoint, or fails the release when the key is absent. +func (g *DefaultObjectsGate) staleCredentials(ctx context.Context, src *corev1.Secret) (string, error) { + if g.CredentialsCOSISecretName == "" || g.CredentialsHelmRelease.Name == "" || g.CredentialsHelmRelease.Namespace == "" { + return "", nil + } + if _, ok := src.Data["endpoint"]; !ok || g.CredentialsCOSISecretName == g.Config.SourceSecretName { + return "", nil + } + cosi := &corev1.Secret{} + key := types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.CredentialsCOSISecretName} + if err := g.Client.Get(ctx, key, cosi); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", fmt.Errorf("get COSI Secret %s: %w", key, err) + } + var info struct { + Spec struct { + SecretS3 struct { + Endpoint string `json:"endpoint"` + } `json:"secretS3"` + } `json:"spec"` + } + if err := json.Unmarshal(cosi.Data["BucketInfo"], &info); err != nil { + return "", fmt.Errorf("parse BucketInfo of COSI Secret %s: %w", key, err) + } + want := strings.TrimPrefix(strings.TrimPrefix(info.Spec.SecretS3.Endpoint, "https://"), "http://") + have := string(src.Data["endpoint"]) + if want == "" || have == want { + return "", nil + } + return fmt.Sprintf("Secret/%s (endpoint %q, COSI Secret has %q)", g.Config.SourceSecretName, have, want), nil +} + +// renderedEndpointFor mirrors the "backupstrategy-controller.endpoint" helper +// for a provisioned bucket: the credentials Secret's endpoint, trimmed, with +// any scheme replaced by https://. It is the value the chart stamps as +// endpointAnnotation. An absent or empty endpoint makes the helper fall back +// to backupStorage.endpoint, which the gate cannot see, so it returns "". +func renderedEndpointFor(raw []byte) string { + if len(raw) == 0 { + return "" + } + host := strings.TrimSpace(string(raw)) + return "https://" + strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://") } // reconcileCredentials handles the one object the gate cannot check the same @@ -419,7 +533,7 @@ func (g *DefaultObjectsGate) force(ctx context.Context, target types.NamespacedN // annotations would be ignored, so skip the patch entirely and // leave the force counter alone — a climbing counter must keep // meaning "the forced render is not producing the objects". - log.FromContext(ctx).Info("skipped forcing a suspended HelmRelease; missing default backup objects will not be repaired until it is resumed", + log.FromContext(ctx).Info("skipped forcing a suspended HelmRelease; missing or stale default backup objects will not be repaired until it is resumed", "helmRelease", target.String(), "missing", missing) *last = now return missing, false, nil @@ -431,16 +545,42 @@ func (g *DefaultObjectsGate) force(ctx context.Context, target types.NamespacedN return missing, true, nil } -// missingObjects returns a sorted, de-duplicated list of the +// defaultObjectsState is what inspectObjects found: the objects that do not +// exist, and the existing ones rendered with an endpoint other than the one +// the chart would render now. +type defaultObjectsState struct { + missing []string + stale []string +} + +// staleEndpoint reports an existing object rendered with another endpoint +// than want, formatted for logs. An empty want disables the comparison. An +// object without the annotation (rendered by a chart that predates it) stays +// stale until a render stamps it: requesting one does not prove it landed. +func (g *DefaultObjectsGate) staleEndpoint(state *defaultObjectsState, key string, obj *unstructured.Unstructured, want string) { + if want == "" { + return + } + have, ok := obj.GetAnnotations()[endpointAnnotation] + switch { + case !ok: + state.stale = append(state.stale, key+" (no endpoint annotation)") + case have != want: + state.stale = append(state.stale, fmt.Sprintf("%s (endpoint %q, want %q)", key, have, want)) + } +} + +// inspectObjects returns a sorted, de-duplicated list of the // "/" objects the BackupClass references that do not exist, -// plus the Velero BSL when a Velero namespace is configured. +// plus the Velero BSL when a Velero namespace is configured, and, unsorted, +// the existing ones whose endpointAnnotation differs from want. // // An unmappable group/kind (the CRD is not installed) is NOT reported as // missing: no Helm re-render can create an object whose CRD is absent, and // counting it would keep the gate forcing upgrades forever. -func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass) ([]string, error) { +func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass, want string) (defaultObjectsState, error) { seen := map[string]struct{}{} - var missing []string + var state defaultObjectsState for _, strategy := range backupClass.Spec.Strategies { name := strategy.StrategyRef.Name @@ -472,17 +612,18 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba if meta.IsNoMatchError(err) { continue } - return nil, fmt.Errorf("map %s/%s: %w", group, kind, err) + return defaultObjectsState{}, fmt.Errorf("map %s/%s: %w", group, kind, err) } // Strategy CRs are cluster-scoped; Namespace("") is correct for // both scopes on the dynamic client. - _, err = g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{}) + obj, err := g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{}) switch { case err == nil: + g.staleEndpoint(&state, key, obj, want) case apierrors.IsNotFound(err): - missing = append(missing, key) + state.missing = append(state.missing, key) default: - return nil, fmt.Errorf("get %s %s: %w", kind, name, err) + return defaultObjectsState{}, fmt.Errorf("get %s %s: %w", kind, name, err) } } @@ -496,24 +637,26 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba mapping, err := g.RESTMapping(schema.GroupKind{Group: backupStorageLocationGVR.Group, Kind: "BackupStorageLocation"}) switch { case err == nil: - _, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) + key := fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName) + obj, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) switch { case getErr == nil: + g.staleEndpoint(&state, key, obj, want) case apierrors.IsNotFound(getErr): - missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)) + state.missing = append(state.missing, key) default: - return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr) + return defaultObjectsState{}, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr) } case meta.IsNoMatchError(err): // Velero API not served (e.g. Velero uninstalled after bootstrap): // no Helm re-render can create the BSL, so it is not "missing". default: - return nil, fmt.Errorf("map BackupStorageLocation: %w", err) + return defaultObjectsState{}, fmt.Errorf("map BackupStorageLocation: %w", err) } } - sort.Strings(missing) - return missing, nil + sort.Strings(state.missing) + return state, nil } // forceHelmRelease stamps BOTH reconcile.fluxcd.io/forceAt and diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index f5cbb5489..e287c9872 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -2,6 +2,9 @@ package backupcontroller import ( "context" + "errors" + "slices" + "strings" "testing" "time" @@ -18,6 +21,7 @@ import ( k8stesting "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) @@ -110,14 +114,60 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { } } +// renderedEndpoint is what the chart stamps on every default object for the +// endpoint sourceSecret carries: the steady state after a render that saw it. +const renderedEndpoint = "https://s3.example.com" + func strategyObject(kind, name string) *unstructured.Unstructured { u := &unstructured.Unstructured{} u.SetAPIVersion(strategyAPIGroup + "/v1alpha1") u.SetKind(kind) u.SetName(name) + u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint}) return u } +func withEndpoint(u *unstructured.Unstructured, endpoint string) *unstructured.Unstructured { + u.SetAnnotations(map[string]string{endpointAnnotation: endpoint}) + return u +} + +// unannotated is an object rendered by a chart that predates the endpoint +// annotation. +func unannotated(u *unstructured.Unstructured) *unstructured.Unstructured { + u.SetAnnotations(nil) + return u +} + +// allDefaultObjects is every object cozy-default routes to, plus the BSL, as +// the chart renders them for sourceSecret's endpoint. +func allDefaultObjects() []runtime.Object { + return []runtime.Object{ + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + bslObject(), + } +} + +// cosiSecret is the Secret the COSI sidecar writes for the platform bucket's +// BucketAccess, which user-credentials.yaml renders the credentials Secret +// from. The BucketInfo document is one read off a live cluster, with the keys +// and the host replaced. +func cosiSecret(endpoint string) *corev1.Secret { + bucketInfo := `{"metadata": {"name": "bc-b8e30337-1821-418f-a4dd-163e8016aaee", "creationTimestamp": null}, ` + + `"spec": {"bucketName": "bucket-1a2b", "authenticationType": "KEY", ` + + `"secretS3": {"endpoint": "` + endpoint + `", "region": "", "accessKeyID": "AK", "accessSecretKey": "SK"}, ` + + `"secretAzure": null, "protocols": ["s3"]}}` + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system"}, + Data: map[string][]byte{"BucketInfo": []byte(bucketInfo)}, + } +} + func helmReleaseObject() *unstructured.Unstructured { u := &unstructured.Unstructured{} u.SetAPIVersion("helm.toolkit.fluxcd.io/v2") @@ -163,9 +213,10 @@ func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) Namespace: "tenant-root", Name: "bucket-cozy-backups-system", }, - VeleroNamespace: "cozy-velero", - MinForceInterval: 5 * time.Minute, - now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + CredentialsCOSISecretName: "bucket-cozy-backups-system", + VeleroNamespace: "cozy-velero", + MinForceInterval: 5 * time.Minute, + now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, } g.Interface = dyn g.RESTMapper = gateRESTMapper() @@ -193,6 +244,7 @@ func bslObject() *unstructured.Unstructured { u.SetKind("BackupStorageLocation") u.SetNamespace("cozy-velero") u.SetName(defaultBackupStorageLocationName) + u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint}) return u } @@ -259,7 +311,7 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) { // the cluster. func TestCheckNoopWhenAllPresent(t *testing.T) { g, dyn := newGate(t, - []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), strategyObject("MariaDB", "cozy-default-mariadb"), @@ -550,10 +602,11 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { bslObject(), ) - missing, err := g.missingObjects(context.Background(), bc) + state, err := g.inspectObjects(context.Background(), bc, "") if err != nil { - t.Fatalf("missingObjects: %v", err) + t.Fatalf("inspectObjects: %v", err) } + missing := state.missing if len(missing) != 0 { t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing) } @@ -579,10 +632,11 @@ func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) { ) g.VeleroNamespace = "" - missing, err := g.missingObjects(context.Background(), bc) + state, err := g.inspectObjects(context.Background(), bc, "") if err != nil { - t.Fatalf("missingObjects: %v", err) + t.Fatalf("inspectObjects: %v", err) } + missing := state.missing if len(missing) != 0 { t.Fatalf("missing = %v, want none when Velero is disabled", missing) } @@ -630,10 +684,11 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { } g.RESTMapper = m - missing, err := g.missingObjects(context.Background(), bc) + state, err := g.inspectObjects(context.Background(), bc, "") if err != nil { - t.Fatalf("missingObjects: %v", err) + t.Fatalf("inspectObjects: %v", err) } + missing := state.missing if len(missing) != 0 { t.Fatalf("missing = %v, want none when the Velero API is absent", missing) } @@ -744,3 +799,525 @@ func TestCheckToleratesAbsentHelmRelease(t *testing.T) { t.Error("objects should still be reported as missing") } } + +// TestCheckForcesCredentialsReleaseWhenEndpointStale covers the credentials +// Secret outliving the endpoint it was rendered from. The COSI sidecar writes +// BucketInfo once per BucketAccess, so a re-granted BucketAccess (after a +// publishing.host change) carries the new endpoint while the credentials +// Secret -system rendered earlier keeps the old one: the Secret +// exists, so the missing-Secret path never fires, and every consumer keeps +// the dead host. +func TestCheckForcesCredentialsReleaseWhenEndpointStale(t *testing.T) { + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()}, + objs..., + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced { + t.Fatal("expected the bucket release forced while its credentials Secret carries a stale endpoint") + } + if len(missing) != 1 || !strings.HasPrefix(missing[0], "Secret/bucket-cozy-backups-system-credentials") { + t.Fatalf("missing = %v, want the credentials Secret", missing) + } + if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("bucket release not forced: %v", ann) + } + // The objects were rendered from the stale Secret and agree with it; + // re-rendering them before the Secret is fixed would bake the old + // endpoint in again. + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("own HelmRelease forced while the credentials Secret is stale: %v", ann) + } +} + +// TestCheckCredentialsEndpointComparison pins the comparison against what +// user-credentials.yaml writes for a given BucketInfo endpoint (the scheme +// stripped, nothing else), so a Secret the chart rendered from the current +// BucketInfo never counts as stale and the bucket release is not forced on +// every throttle window. +func TestCheckCredentialsEndpointComparison(t *testing.T) { + for _, tc := range []struct { + name string + cosi string + wantForce bool + }{ + {"https scheme stripped", "https://s3.example.com", false}, + {"http scheme stripped", "http://s3.example.com", false}, + {"bare host", "s3.example.com", false}, + {"other host", "https://s3.new.example.com", true}, + {"other port", "https://s3.example.com:8443", true}, + // A forced render from a BucketInfo with no endpoint would write an + // empty endpoint into the Secret every driver reads (or, with the key + // absent, fail the bucket release's render), never a working one. + {"no endpoint in BucketInfo", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret(tc.cosi), cozyDefaultBackupClass()}, + objs..., + ) + if _, _, err := g.Check(context.Background()); err != nil { + t.Fatalf("Check: %v", err) + } + forced := credentialsForceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] != "" + if forced != tc.wantForce { + t.Fatalf("bucket release forced = %v, want %v", forced, tc.wantForce) + } + }) + } +} + +// TestCheckSkipsCredentialsComparisonWithoutCOSISecret covers the window in +// which the BucketAccess is being re-granted: the COSI Secret is gone and +// there is nothing to compare against, which is not evidence of staleness. +func TestCheckSkipsCredentialsComparisonWithoutCOSISecret(t *testing.T) { + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + objs..., + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil || forced || len(missing) != 0 { + t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release forced with no COSI Secret to compare against: %v", ann) + } +} + +// TestCheckSkipsCredentialsComparisonWhenUnconfigured pins both switches that +// turn the comparison off: no COSI Secret name, and no bucket release (external +// S3, where the Secret is admin-managed and no BucketInfo backs it). +func TestCheckSkipsCredentialsComparisonWhenUnconfigured(t *testing.T) { + for _, tc := range []struct { + name string + apply func(g *DefaultObjectsGate) + }{ + {"no COSI Secret name", func(g *DefaultObjectsGate) { g.CredentialsCOSISecretName = "" }}, + {"no bucket release", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease = types.NamespacedName{} }}, + {"no bucket release name", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Name = "" }}, + {"no bucket release namespace", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Namespace = "" }}, + } { + t.Run(tc.name, func(t *testing.T) { + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()}, + objs..., + ) + tc.apply(g) + // The real API server rejects a Get with an empty name; the fake + // answers NotFound, which would hide a lookup of an unset name. + g.Client = interceptor.NewClient(g.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + missing, forced, err := g.Check(context.Background()) + if err != nil || forced || len(missing) != 0 { + t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release forced with the comparison disabled: %v", ann) + } + }) + } +} + +// TestCheckReportsMalformedCOSISecret pins that a COSI Secret the gate cannot +// read is an error the check counts, not a silent "not stale". +func TestCheckReportsMalformedCOSISecret(t *testing.T) { + defaultObjectsCheckErrors.Reset() + bad := cosiSecret("") + bad.Data["BucketInfo"] = []byte("{not json") + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), bad, cozyDefaultBackupClass()}, objs...) + + if _, forced, err := g.Check(context.Background()); err == nil || forced { + t.Fatalf("forced = %v, err = %v; want an error", forced, err) + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("check errors = %v, want 1", got) + } +} + +// TestCheckForcesOwnReleaseWhenObjectEndpointStale is the second half of a +// publishing.host change: the credentials Secret now carries the new +// endpoint, but the BSL and the default Strategies baked the old one at the +// last render of this chart and nothing re-renders them, so every backup +// keeps going to the dead host. +func TestCheckForcesOwnReleaseWhenObjectEndpointStale(t *testing.T) { + for _, tc := range []struct { + name string + stale *unstructured.Unstructured + key string + }{ + {"strategy", strategyObject("CNPG", "cozy-default-cnpg"), "CNPG/cozy-default-cnpg"}, + {"BSL", bslObject(), "BackupStorageLocation/cozy-default"}, + } { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + for _, o := range allDefaultObjects() { + u := o.(*unstructured.Unstructured) + if u.GetKind() == tc.stale.GetKind() && u.GetName() == tc.stale.GetName() { + withEndpoint(u, "https://s3.old.example.com") + } + objs = append(objs, u) + } + objs = append(objs, helmReleaseObject(), credentialsHelmReleaseObject()) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()}, + objs..., + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced { + t.Fatal("expected this chart's release forced while an object carries a stale endpoint") + } + if len(missing) != 1 || !strings.HasPrefix(missing[0], tc.key+" ") { + t.Fatalf("missing = %v, want only %s", missing, tc.key) + } + if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("own HelmRelease not forced: %v", ann) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release forced for a stale object: %v", ann) + } + }) + } +} + +// TestCheckIgnoresObjectEndpointOnExternalS3 pins that the comparison is off +// when no bucket release is configured: with provisionBucket=false the chart +// renders backupStorage.endpoint, not the Secret's, so the two legitimately +// differ and forcing would loop every MinForceInterval. +func TestCheckIgnoresObjectEndpointOnExternalS3(t *testing.T) { + objs := []runtime.Object{helmReleaseObject()} + for _, o := range allDefaultObjects() { + objs = append(objs, withEndpoint(o.(*unstructured.Unstructured), "https://minio.example.internal:9000")) + } + g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...) + g.CredentialsHelmRelease = types.NamespacedName{} + + missing, forced, err := g.Check(context.Background()) + if err != nil || forced || len(missing) != 0 { + t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("own HelmRelease forced on external S3: %v", ann) + } +} + +// TestCheckIgnoresObjectEndpointWithoutSecretEndpoint covers a credentials +// Secret with no endpoint key: the chart then renders its values fallback, +// which the gate cannot reproduce, so there is nothing to compare against. +func TestCheckIgnoresObjectEndpointWithoutSecretEndpoint(t *testing.T) { + src := sourceSecret("bucket-1a2b") + delete(src.Data, "endpoint") + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, []client.Object{src, cozyDefaultBackupClass()}, objs...) + + if _, forced, err := g.Check(context.Background()); err != nil || forced { + t.Fatalf("forced = %v, err = %v; want a no-op", forced, err) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("own HelmRelease forced with no endpoint to compare against: %v", ann) + } +} + +// TestCheckKeepsForcingForUnannotatedObjects covers objects rendered by a +// chart that predates the endpoint annotation. Their endpoint is unknown, so +// they count as stale until a render stamps the annotation. Having requested +// a render proves nothing: one that failed leaves them unannotated, and the +// next unthrottled check must force again rather than fall silent, so the +// force counter keeps reporting the render that does not land. +func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + objs := []runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()} + for _, o := range allDefaultObjects() { + objs = append(objs, unannotated(o.(*unstructured.Unstructured))) + } + g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...) + g.lastForce = base.Add(-time.Minute) + g.now = func() time.Time { return base } + + if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 { + t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err) + } + + g.now = func() time.Time { return base.Add(5 * time.Minute) } + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("first unthrottled Check: forced = %v, err = %v; want forced", forced, err) + } + first := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] + + g.now = func() time.Time { return base.Add(time.Hour) } + missing, forced, err := g.Check(context.Background()) + if err != nil || !forced || len(missing) != 7 { + t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err) + } + if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first { + t.Errorf("not forced again for unannotated objects: forceAt still %q", first) + } +} + +// TestRenderedEndpointMatchesTheChart pins renderedEndpointFor to the +// "backupstrategy-controller.endpoint" helper, which the chart stamps as the +// annotation: trim, strip a scheme, force https://. Any divergence makes the +// gate see every object as stale and force this release forever. +func TestRenderedEndpointMatchesTheChart(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"s3.example.com", "https://s3.example.com"}, + {"s3.example.com\n", "https://s3.example.com"}, + {" s3.example.com ", "https://s3.example.com"}, + {"https://s3.example.com", "https://s3.example.com"}, + {"http://s3.example.com:8333", "https://s3.example.com:8333"}, + {"s3.example.com/prefix/", "https://s3.example.com/prefix/"}, + // No endpoint in the Secret: the chart renders its values fallback, + // which the gate cannot reproduce, so it compares nothing. + {"", ""}, + // Present but blank: the helper's lookup sees a non-empty value and + // renders the bare scheme. + {"\n", "https://"}, + } { + if got := renderedEndpointFor([]byte(tc.in)); got != tc.want { + t.Errorf("renderedEndpointFor(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestForcedReleasesForNamesEachRenderingRelease pins the releases the log +// line names: the credentials Secret, missing or stale, is repaired by the +// bucket release, everything else by this chart's own, and both are named when +// one check forced both. +func TestForcedReleasesForNamesEachRenderingRelease(t *testing.T) { + g, _ := newGate(t, nil) + creds, own := g.CredentialsHelmRelease.String(), g.HelmRelease.String() + for _, tc := range []struct { + missing []string + want []string + }{ + {[]string{"Secret/bucket-cozy-backups-system-credentials"}, []string{creds}}, + {[]string{`Secret/bucket-cozy-backups-system-credentials (endpoint "s3.old", COSI Secret has "s3.new")`}, []string{creds}}, + {[]string{"Secret/bucket-cozy-backups-system-credentials-other"}, []string{own}}, + {[]string{`CNPG/cozy-default-cnpg (endpoint "https://s3.old", want "https://s3.new")`, "Etcd/cozy-default-etcd"}, []string{own}}, + {[]string{`Secret/bucket-cozy-backups-system-credentials (endpoint "s3.old", COSI Secret has "s3.new")`, "CNPG/cozy-default-cnpg"}, []string{creds, own}}, + } { + if got := g.forcedReleasesFor(tc.missing); !slices.Equal(got, tc.want) { + t.Errorf("forcedReleasesFor(%v) = %v, want %v", tc.missing, got, tc.want) + } + } + // External S3: no bucket release, so nothing but this chart's own. + g.CredentialsHelmRelease = types.NamespacedName{} + if got := g.forcedReleasesFor([]string{"Secret/bucket-cozy-backups-system-credentials"}); !slices.Equal(got, []string{own}) { + t.Errorf("forcedReleasesFor without a bucket release = %v, want [%s]", got, own) + } +} + +// TestCheckSkipsCredentialsComparisonWithoutDerivedSecret covers a +// systemSecretName the bucket chart does not render: the same name as the COSI +// Secret (the projector also reads a raw COSI Secret) or a Secret without the +// endpoint key. Comparing it would report it stale on every tick, or fail the +// check, and the missing objects would never be repaired. +func TestCheckSkipsCredentialsComparisonWithoutDerivedSecret(t *testing.T) { + for _, tc := range []struct { + name string + apply func(g *DefaultObjectsGate, src *corev1.Secret) + }{ + {"COSI Secret is the source", func(g *DefaultObjectsGate, src *corev1.Secret) { + g.CredentialsCOSISecretName = g.Config.SourceSecretName + src.Data["BucketInfo"] = cosiSecret("https://s3.new.example.com").Data["BucketInfo"] + }}, + {"source without endpoint", func(_ *DefaultObjectsGate, src *corev1.Secret) { + delete(src.Data, "endpoint") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + src := sourceSecret("bucket-1a2b") + var objs []runtime.Object + for _, o := range allDefaultObjects() { + if u := o.(*unstructured.Unstructured); u.GetName() != "cozy-default-cnpg" { + objs = append(objs, u) + } + } + objs = append(objs, helmReleaseObject(), credentialsHelmReleaseObject()) + g, dyn := newGate(t, nil, objs...) + tc.apply(g, src) + g.Client = fake.NewClientBuilder().WithScheme(gateScheme()). + WithObjects(src, cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()).Build() + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced || len(missing) != 1 || missing[0] != "CNPG/cozy-default-cnpg" { + t.Fatalf("missing = %v, forced = %v; want the missing strategy repaired", missing, forced) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release forced: %v", ann) + } + }) + } +} + +// TestCheckRepairsMissingObjectsWhileCredentialsUnresolved pins that a stale +// or unreadable credentials comparison does not cost the repair of missing +// objects the gate made before it existed. The comparison only withholds the +// endpoint check, whose answer would be rendered from the same stale Secret. +func TestCheckRepairsMissingObjectsWhileCredentialsUnresolved(t *testing.T) { + malformed := cosiSecret("") + malformed.Data["BucketInfo"] = []byte("{not json") + for _, tc := range []struct { + name string + cosi *corev1.Secret + credsHR *unstructured.Unstructured + wantErr bool + wantCredsHRAt bool + }{ + {"stale credentials", cosiSecret("https://s3.new.example.com"), credentialsHelmReleaseObject(), false, true}, + {"stale credentials, bucket release suspended", cosiSecret("https://s3.new.example.com"), suspended(credentialsHelmReleaseObject()), false, false}, + {"malformed COSI Secret", malformed, credentialsHelmReleaseObject(), true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + defaultObjectsCheckErrors.Reset() + defaultObjectsMissing.Reset() + var objs []runtime.Object + for _, o := range allDefaultObjects() { + if u := o.(*unstructured.Unstructured); u.GetName() != "cozy-default-cnpg" { + objs = append(objs, u) + } + } + objs = append(objs, helmReleaseObject(), tc.credsHR) + g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), tc.cosi, cozyDefaultBackupClass()}, objs...) + + missing, forced, err := g.Check(context.Background()) + if (err != nil) != tc.wantErr { + t.Fatalf("err = %v, want error %v", err, tc.wantErr) + } + if !forced { + t.Error("forced = false, want this chart's release forced for the missing strategy") + } + if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("own HelmRelease not forced for a missing strategy: %v", ann) + } + if got := credentialsForceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] != ""; got != tc.wantCredsHRAt { + t.Errorf("bucket release forced = %v, want %v", got, tc.wantCredsHRAt) + } + found := false + for _, m := range missing { + found = found || m == "CNPG/cozy-default-cnpg" + } + if !found { + t.Errorf("missing = %v, want CNPG/cozy-default-cnpg reported", missing) + } + if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 { + t.Errorf("missing gauge = %v, want 1", got) + } + wantErrs := 0.0 + if tc.wantErr { + wantErrs = 1 + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != wantErrs { + t.Errorf("check errors = %v, want %v", got, wantErrs) + } + }) + } +} + +// TestCheckCountsOneErrorPerCheck pins that a check failing in two places is +// still one check that did not reach a conclusion. +func TestCheckCountsOneErrorPerCheck(t *testing.T) { + defaultObjectsCheckErrors.Reset() + bad := cosiSecret("") + bad.Data["BucketInfo"] = []byte("{not json") + g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), bad}, helmReleaseObject(), credentialsHelmReleaseObject()) + + if _, _, err := g.Check(context.Background()); err == nil { + t.Fatal("Check succeeded with a malformed COSI Secret and no BackupClass") + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("check errors = %v, want 1", got) + } +} + +// TestCheckSurfacesCredentialsPatchFailure pins that a failed force of the +// bucket release is reported, not recorded as done. +func TestCheckSurfacesCredentialsPatchFailure(t *testing.T) { + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()}, + objs..., + ) + dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "bucket-cozy-backups-system", nil) + }) + + if _, forced, err := g.Check(context.Background()); err == nil || forced { + t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err) + } + if !g.lastCredentialsForce.IsZero() { + t.Error("lastCredentialsForce advanced despite the patch failing") + } +} + +// TestCheckForcesStaleCredentialsWhenObjectsUnreadable pins that the +// credentials comparison needs only the two Secrets: a BackupClass or a routed +// object the gate cannot read must not keep the stale endpoint in the +// credentials Secret, which every other driver reads through the projector. +func TestCheckForcesStaleCredentialsWhenObjectsUnreadable(t *testing.T) { + for _, tc := range []struct { + name string + backupClass bool + forbidCNPG bool + }{ + {"BackupClass absent", false, false}, + {"routed Strategy unreadable", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + defaultObjectsCheckErrors.Reset() + ctrlObjs := []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com")} + if tc.backupClass { + ctrlObjs = append(ctrlObjs, cozyDefaultBackupClass()) + } + objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...) + g, dyn := newGate(t, ctrlObjs, objs...) + if tc.forbidCNPG { + dyn.PrependReactor("get", "cnpgs", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: strategyAPIGroup, Resource: "cnpgs"}, "cozy-default-cnpg", nil) + }) + } + + missing, forced, err := g.Check(context.Background()) + if err == nil { + t.Fatal("Check succeeded although the routed objects could not be evaluated") + } + if !forced { + t.Error("forced = false, want the bucket release forced for its stale credentials Secret") + } + if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("bucket release not forced: %v", ann) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("own HelmRelease forced without a conclusive object check: %v", ann) + } + if len(missing) != 1 || !strings.HasPrefix(missing[0], "Secret/bucket-cozy-backups-system-credentials") { + t.Errorf("missing = %v, want the credentials Secret", missing) + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 { + t.Errorf("check errors = %v, want 1", got) + } + }) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 4d88ab8a1..9fd6780db 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -204,7 +205,10 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { - case cond != nil && cond.Status == metav1.ConditionTrue: + case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed: + return r.markBackupJobFailed(ctx, j, mariadbJobFailedMessage(ctx, r.Client, "Backup", mdbBackup.Namespace, mdbBackup.Name, cond)) + + case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete: // Backup completed successfully. Materialise the Cozystack artifact // and finalise the BackupJob. Idempotent: if the artifact already // exists (a previous reconcile created it and then raced on the @@ -231,34 +235,54 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 } return ctrl.Result{}, nil - case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed: - // Terminal failure. The operator marks Complete=False/JobFailed only - // after the underlying Job exhausts its backoff budget, so we can - // fail the BackupJob immediately rather than waiting for the - // driver-side deadline. - message := cond.Message - if message == "" { - message = "k8s.mariadb.com Backup terminal failure" - } - return r.markBackupJobFailed(ctx, j, message) - default: // Still running (Complete=False/JobRunning, Complete=False/JobNotComplete, - // JobSuspended, or no Complete condition yet). Apply a wall-clock - // deadline so a permanently-stuck Backup eventually fails the - // BackupJob instead of pinning it Running forever. + // JobSuspended, a reason this driver does not know, or no Complete + // condition yet). Apply a wall-clock deadline so a permanently-stuck + // Backup eventually fails the BackupJob instead of pinning it Running + // forever. if mariadbBackupDeadlineExceeded(j.Status.StartedAt) { - detail := "no Complete condition observed" - if cond != nil { - detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) - } return r.markBackupJobFailed(ctx, j, fmt.Sprintf( - "k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, detail)) + "k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, mariadbCompleteDetail(cond))) } return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil } } +// mariadbJobFailedMessage names the operator object whose Job failed and +// carries that Job's own Failed condition, because the operator's condition +// message is only "Failed". The operator names the Job after the object. +func mariadbJobFailedMessage(ctx context.Context, c client.Reader, kind, namespace, name string, cond *metav1.Condition) string { + message := fmt.Sprintf("k8s.mariadb.com/%s %s/%s failed (Complete=%s/%s)", kind, namespace, name, cond.Status, cond.Reason) + job := &batchv1.Job{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, job); err != nil { + if apierrors.IsNotFound(err) { + return fmt.Sprintf("%s; Job %s/%s not found", message, namespace, name) + } + return fmt.Sprintf("%s; reading Job %s/%s: %v", message, namespace, name, err) + } + for _, jc := range job.Status.Conditions { + if jc.Type == batchv1.JobFailed && jc.Status == corev1.ConditionTrue { + return fmt.Sprintf("%s; Job %s/%s: %s: %s", message, namespace, name, jc.Reason, jc.Message) + } + } + return fmt.Sprintf("%s; Job %s/%s reports no Failed condition", message, namespace, name) +} + +// mariadbCompleteDetail describes the last Complete condition for a deadline +// failure. The message matters for reason Failed, which the operator sets +// with its own cause ("Error creating Job", "Backup not complete") rather +// than from a Job. +func mariadbCompleteDetail(cond *metav1.Condition) string { + if cond == nil { + return "no Complete condition observed" + } + if cond.Message == "" { + return fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) + } + return fmt.Sprintf("Complete=%s/%s: %s", cond.Status, cond.Reason, cond.Message) +} + // mariadbBackupDeadlineExceeded reports whether enough wall-clock time has // elapsed since the BackupJob started that we should give up on a stuck // operator Backup. Returns false when StartedAt is nil so the very first @@ -577,7 +601,10 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { - case cond != nil && cond.Status == metav1.ConditionTrue: + case cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed: + return r.markRestoreJobFailed(ctx, restoreJob, mariadbJobFailedMessage(ctx, r.Client, "Restore", mdbRestore.Namespace, mdbRestore.Name, cond)) + + case cond != nil && cond.Status == metav1.ConditionTrue && cond.Reason == mariadbtypes.ConditionReasonJobComplete: now := metav1.Now() restoreJob.Status.CompletedAt = &now restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded @@ -592,23 +619,12 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } return ctrl.Result{}, nil - case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed: - message := cond.Message - if message == "" { - message = fmt.Sprintf("k8s.mariadb.com/Restore %s/%s terminal failure", mdbRestore.Namespace, mdbRestore.Name) - } - return r.markRestoreJobFailed(ctx, restoreJob, message) - default: deadline := options.effectiveRestoreDeadline() if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline { - detail := "no Complete condition observed" - if cond != nil { - detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason) - } return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf( "k8s.mariadb.com/Restore did not complete within %s (%s; override via spec.options.restoreTimeoutSeconds)", - deadline, detail)) + deadline, mariadbCompleteDetail(cond))) } return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil } diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index 0e4f63b5e..f03b67d94 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -4,12 +4,15 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "strings" "testing" "time" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -17,6 +20,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" @@ -867,6 +871,302 @@ func TestReconcileMariaDBRestore_KindCheckedBeforeSrcBackupGet(t *testing.T) { } } +// The operator reports a failed backing Job as Complete=True with reason +// JobFailed (SetCompleteWithJob in mariadb-operator 25.10.2), so the +// Complete status alone does not tell success from failure. +var mariadbCompleteOutcomeCases = []struct { + name string + noCondition bool + status metav1.ConditionStatus + reason string + message string + expired bool + failedJob bool + wantFailed bool + wantDone bool + wantMessage []string +}{ + {name: "Complete=True/JobComplete succeeds", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success", wantDone: true}, + {name: "Complete=True/JobFailed fails with the Job's failure", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true, + wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}}, + {name: "Complete=False/JobFailed fails with the Job's failure", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", failedJob: true, wantFailed: true, + wantMessage: []string{"tenant/op", "BackoffLimitExceeded", "Job has reached the specified backoff limit"}}, + {name: "Complete=True/JobFailed with its Job gone fails", status: metav1.ConditionTrue, reason: mariadbtypes.ConditionReasonJobFailed, message: "Failed", wantFailed: true, + wantMessage: []string{"tenant/op", "Job tenant/op not found"}}, + {name: "Complete=True with an unknown reason keeps waiting", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something"}, + {name: "Complete=False/JobComplete keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobComplete, message: "Success"}, + {name: "Complete=False/JobRunning keeps waiting", status: metav1.ConditionFalse, reason: mariadbtypes.ConditionReasonJobRunning, message: "Running"}, + {name: "Complete=False/Failed keeps waiting", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job"}, + {name: "no Complete condition keeps waiting", noCondition: true}, + {name: "no Complete condition past the deadline fails", noCondition: true, expired: true, wantFailed: true, + wantMessage: []string{"no Complete condition observed"}}, + {name: "Complete=True with an unknown reason past the deadline fails", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something", expired: true, wantFailed: true, + wantMessage: []string{"Complete=True/SomethingElse"}}, + {name: "Complete=False/Failed past the deadline carries the operator's message", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job", expired: true, wantFailed: true, + wantMessage: []string{"Complete=False/Failed: Error creating Job"}}, +} + +func mariadbCompleteConditions(noCondition bool, status metav1.ConditionStatus, reason, message string) []metav1.Condition { + if noCondition { + return nil + } + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: metav1.Now(), + }} +} + +// mariadbOutcomeStartedAt returns a StartedAt that is fresh, or older than +// both the backup and the default restore deadline. +func mariadbOutcomeStartedAt(expired bool) *metav1.Time { + if !expired { + now := metav1.Now() + return &now + } + past := metav1.NewTime(time.Now().Add(-2 * max(mariadbDefaultBackupDeadline, mariadbDefaultRestoreDeadline))) + return &past +} + +// mariadbOutcomeObjects adds the operator's Job, named after the operator +// Backup or Restore, failed the way the Job controller fails it once the +// backoff limit is spent. +func mariadbOutcomeObjects(failedJob bool, objs ...client.Object) []client.Object { + if !failedJob { + return objs + } + return append(objs, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Reason: batchv1.JobReasonBackoffLimitExceeded, + Message: "Job has reached the specified backoff limit", + }}}, + }) +} + +func assertMariaDBFailureMessage(t *testing.T, message string, want []string) { + t.Helper() + if strings.Contains(message, "logs") { + t.Errorf("failure message must not send the reader to Job logs, which are gone with the deleted pod: %q", message) + } + for _, w := range want { + if !strings.Contains(message, w) { + t.Errorf("failure message must contain %q, got %q", w, message) + } + } +} + +func TestReconcileMariaDBBackup_CompleteConditionOutcome(t *testing.T) { + strategyAPIGroup := strategyv1alpha1.GroupVersion.Group + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyAPIGroup, + Kind: strategyv1alpha1.MariaDBStrategyKind, + Name: "mdb-strategy", + }, + } + + for _, tc := range mariadbCompleteOutcomeCases { + t.Run(tc.name, func(t *testing.T) { + seed := newMariaDBBackupJob("bj", "tenant") + seed.Status.StartedAt = mariadbOutcomeStartedAt(tc.expired) + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp(seed.Spec.ApplicationRef.Name)}, + } + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "op", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: seed.Name, + backupsv1alpha1.OwningJobNamespaceLabel: seed.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)}, + } + c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, seed, newMariaDBApp(seed.Spec.ApplicationRef.Name, "tenant"), + strategy, operatorMariaDB, operatorBackup)...) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)} + + bj := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), bj); err != nil { + t.Fatalf("seed Get: %v", err) + } + res, err := r.reconcileMariaDB(context.Background(), bj, resolved) + if err != nil { + t.Fatalf("reconcileMariaDB: %v", err) + } + + stored := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), stored); err != nil { + t.Fatalf("persisted Get: %v", err) + } + artifacts := &backupsv1alpha1.BackupList{} + if err := c.List(context.Background(), artifacts, client.InNamespace("tenant")); err != nil { + t.Fatalf("list Backups: %v", err) + } + switch { + case tc.wantFailed: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("expected Phase=Failed, got %q", stored.Status.Phase) + } + assertMariaDBFailureMessage(t, stored.Status.Message, tc.wantMessage) + if stored.Status.BackupRef != nil || len(artifacts.Items) != 0 { + t.Errorf("a failed run must not produce a Backup: ref=%v artifacts=%d", stored.Status.BackupRef, len(artifacts.Items)) + } + case tc.wantDone: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Errorf("expected Phase=Succeeded, got %q (message %q)", stored.Status.Phase, stored.Status.Message) + } + if stored.Status.BackupRef == nil || len(artifacts.Items) != 1 { + t.Errorf("a successful run must produce exactly one Backup: ref=%v artifacts=%d", stored.Status.BackupRef, len(artifacts.Items)) + } + default: + if stored.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning { + t.Errorf("expected Phase=Running, got %q", stored.Status.Phase) + } + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the run is in progress, got %+v", res) + } + if len(artifacts.Items) != 0 { + t.Errorf("an unfinished run must not produce a Backup, got %d", len(artifacts.Items)) + } + } + }) + } +} + +func TestMariaDBJobFailedMessage_JobUnreadable(t *testing.T) { + cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed} + readErr := apierrors.NewForbidden(batchv1.Resource("jobs"), "op", errors.New("rbac")) + c := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*batchv1.Job); ok { + return readErr + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + + got := mariadbJobFailedMessage(context.Background(), c, "Backup", "tenant", "op", cond) + for _, want := range []string{"k8s.mariadb.com/Backup tenant/op failed (Complete=True/JobFailed)", "reading Job tenant/op", readErr.Error()} { + if !strings.Contains(got, want) { + t.Errorf("message must contain %q, got %q", want, got) + } + } +} + +func TestMariaDBJobFailedMessage_JobWithoutFailedCondition(t *testing.T) { + cond := &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, Reason: mariadbtypes.ConditionReasonJobFailed} + c := newMariaDBStrategyTestClient(t, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue, Reason: batchv1.JobReasonPodFailurePolicy}, + {Type: batchv1.JobFailed, Status: corev1.ConditionFalse, Reason: batchv1.JobReasonBackoffLimitExceeded}, + }}, + }) + + got := mariadbJobFailedMessage(context.Background(), c, "Restore", "tenant", "op", cond) + if !strings.Contains(got, "Job tenant/op reports no Failed condition") || + strings.Contains(got, batchv1.JobReasonBackoffLimitExceeded) || strings.Contains(got, batchv1.JobReasonPodFailurePolicy) { + t.Errorf("a Failed condition that is not True must not be reported as the cause, got %q", got) + } +} + +func TestMariaDBCompleteDetail(t *testing.T) { + for _, tc := range []struct { + cond *metav1.Condition + want string + }{ + {nil, "no Complete condition observed"}, + {&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed", Message: "Backup not complete"}, "Complete=False/Failed: Backup not complete"}, + {&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed"}, "Complete=False/Failed"}, + } { + if got := mariadbCompleteDetail(tc.cond); got != tc.want { + t.Errorf("mariadbCompleteDetail(%+v) = %q, want %q", tc.cond, got, tc.want) + } + } +} + +func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) { + apps := mariadbapp.GroupName + + for _, tc := range mariadbCompleteOutcomeCases { + t.Run(tc.name, func(t *testing.T) { + cozyBackup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-cozy-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + DriverMetadata: map[string]string{mariadbBackupNameKey: "src-op-bk"}, + }, + } + seed := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: cozyBackup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: mariadbOutcomeStartedAt(tc.expired), Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + operatorRestore := &mariadbtypes.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "op", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: seed.Name, + backupsv1alpha1.OwningJobNamespaceLabel: seed.Namespace, + }, + }, + Status: mariadbtypes.RestoreStatus{Conditions: mariadbCompleteConditions(tc.noCondition, tc.status, tc.reason, tc.message)}, + } + c := newMariaDBStrategyTestClient(t, mariadbOutcomeObjects(tc.failedJob, cozyBackup, seed, operatorRestore, + &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-op-bk"}}, + &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}})...) + r := &RestoreJobReconciler{Client: c, Recorder: record.NewFakeRecorder(10)} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), rj); err != nil { + t.Fatalf("seed Get: %v", err) + } + res, err := r.reconcileMariaDBRestore(context.Background(), rj, cozyBackup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + + stored := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(seed), stored); err != nil { + t.Fatalf("persisted Get: %v", err) + } + switch { + case tc.wantFailed: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("expected Phase=Failed, got %q", stored.Status.Phase) + } + assertMariaDBFailureMessage(t, stored.Status.Message, tc.wantMessage) + case tc.wantDone: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Errorf("expected Phase=Succeeded, got %q (message %q)", stored.Status.Phase, stored.Status.Message) + } + default: + if stored.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning { + t.Errorf("expected Phase=Running, got %q", stored.Status.Phase) + } + if res.RequeueAfter == 0 { + t.Errorf("expected a requeue while the run is in progress, got %+v", res) + } + } + }) + } +} + // --------------------------------------------------------------------------- // Test helpers // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/mariadbtypes/types.go b/internal/backupcontroller/mariadbtypes/types.go index 7bc5b03a1..61fc71019 100644 --- a/internal/backupcontroller/mariadbtypes/types.go +++ b/internal/backupcontroller/mariadbtypes/types.go @@ -27,16 +27,20 @@ const ( GroupName = "k8s.mariadb.com" Version = "v1alpha1" - // Operator-side condition type set on Backup and Restore once the run - // terminates. Status=True means success; Status=False with Reason - // explains the failure or in-progress state. + // Operator-side condition type set on Backup and Restore. Status=True + // means the backing Job terminated, successfully or not; only the + // Reason tells the two apart (SetCompleteWithJob in mariadb-operator). ConditionTypeComplete = "Complete" // Reason values the upstream operator emits on the Complete condition. // JobComplete -> Status=True (run succeeded). - // JobFailed -> Status=False (terminal failure of the backing Job). + // JobFailed -> Status=True (terminal failure of the backing Job). // JobRunning -> Status=False (still in progress). // JobSuspended -> Status=False (admin-suspended via spec). + // The operator also writes reason Failed with Status=False and its own + // message ("Error creating Job", "Backup not complete") when a run fails + // outside its Job. It retries those, so the driver keeps waiting on + // Failed until its deadline rather than failing the run at once. ConditionReasonJobComplete = "JobComplete" ConditionReasonJobFailed = "JobFailed" ConditionReasonJobRunning = "JobRunning" @@ -183,7 +187,7 @@ type S3TLS struct { } // BackupStatus mirrors Backup.status. The operator expresses success -// via condition type "Complete" with status=True. +// via condition type "Complete" with reason JobComplete. type BackupStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } @@ -217,7 +221,7 @@ type RestoreSpec struct { } // RestoreStatus mirrors Restore.status. Same condition convention as -// Backup: type=Complete, Status=True on success. +// Backup: type=Complete, reason JobComplete on success. type RestoreStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go index 0ead1bc58..9caa218a7 100644 --- a/internal/backupcontroller/psmdbstrategy_controller.go +++ b/internal/backupcontroller/psmdbstrategy_controller.go @@ -1376,9 +1376,10 @@ func (r *BackupJobReconciler) createMongoDBBackupArtifact( // annotation (honoured before any apiserver read, so it still frees a Backup // whose operator CR has become unreadable) and a going-away namespace (the psmdb // delete-backup exec cannot run there) each make the driver strip its own -// finalizer and release, leaving the object for a bucket lifecycle policy. This -// bounds only what the driver contributes: a Get that fails with something other -// than NotFound/NoMatchError still requeues (below), and the psmdb operator's own +// finalizer and release, leaving the object to manual cleanup, or to a +// lifecycle rule on the bucket where the object store applies one. This bounds +// only what the driver contributes: a Get that fails with something other than +// NotFound/NoMatchError still requeues (below), and the psmdb operator's own // release-lock finalizer keeps the CR Terminating while the operator is down, // whatever the driver strips. A down operator in a live namespace is likewise // not auto-released: the Backup stays Terminating as a visible signal until an diff --git a/internal/backupcontroller/redis_artifact_size.go b/internal/backupcontroller/redis_artifact_size.go new file mode 100644 index 000000000..49a73d292 --- /dev/null +++ b/internal/backupcontroller/redis_artifact_size.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// redisArtifact locates the RDB snapshot the backup Job uploaded and returns +// an Artifact carrying its s3:// URI and size. The default Redis strategy +// takes its S3 coordinates from the projected backup credentials Secret, so +// the object is looked up there, with the TLS trust the Job's pod applied +// (see redisS3Trust). It returns an artifactProbeError, and no Artifact, +// when the Secret cannot be read or the object is not found: a custom +// strategy that writes elsewhere must not get a URI pointing at an object +// that does not exist. +func (r *BackupJobReconciler) redisArtifact(ctx context.Context, namespace, objectKey string, podSpec *corev1.PodSpec) (*backupsv1alpha1.BackupArtifact, error) { + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) + defer cancel() + + bucket, conn, err := r.projectedS3Connection(ctx, namespace) + if err != nil { + return nil, &artifactProbeError{reason: "the backup credentials Secret is unusable", err: err} + } + if conn.CAPEM, conn.InsecureSkipVerify, err = r.redisS3Trust(ctx, namespace, podSpec); err != nil { + return nil, &artifactProbeError{reason: "the S3 endpoint CA the Job mounted cannot be read", err: err} + } + // The script passes the CA as curl --cacert, which replaces the default + // trust store rather than adding to it. + conn.CAOnly = true + lister, err := r.s3Lister(conn) + if err != nil { + return nil, &artifactProbeError{reason: "the S3 endpoint is not usable", err: err} + } + size, err := lister.StatObject(ctx, bucket, objectKey) + if err != nil { + return nil, &artifactProbeError{reason: "the snapshot could not be found or read in the bucket", err: err} + } + return &backupsv1alpha1.BackupArtifact{ + URI: fmt.Sprintf("s3://%s/%s", bucket, objectKey), + SizeBytes: size, + }, nil +} + +// artifactProbeError carries a fixed reason fit for an Event in the +// application namespace; err may name the platform's endpoint and bucket, +// so it goes to the controller log only. +type artifactProbeError struct { + reason string + err error +} + +func (e *artifactProbeError) Error() string { return e.reason + ": " + e.err.Error() } +func (e *artifactProbeError) Unwrap() error { return e.err } + +// projectedS3Connection reads the backup credentials Secret the controller +// projects into the application namespace. The projector stores the endpoint +// without a scheme, and the strategy scripts reach it over https. It can +// still carry a path, which minio-go refuses. +func (r *BackupJobReconciler) projectedS3Connection(ctx context.Context, namespace string) (string, s3Connection, error) { + var conn s3Connection + name := r.CredentialsConfig.TargetSecretName + if name == "" { + return "", conn, errors.New("no projected credentials Secret is configured") + } + secret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", conn, err + } + value := func(key string) string { return string(secret.Data[key]) } + + bucket := value("bucketName") + conn.AccessKey, conn.SecretKey = value("AWS_ACCESS_KEY_ID"), value("AWS_SECRET_ACCESS_KEY") + if bucket == "" || value("endpoint") == "" || conn.AccessKey == "" || conn.SecretKey == "" { + return "", conn, fmt.Errorf("secret %s/%s lacks bucketName, endpoint or access keys", namespace, name) + } + conn.Endpoint, conn.Secure = value("endpoint"), true + conn.Region = value("region") + return bucket, conn, nil +} + +const ( + redisS3CAVolume = "s3-ca" + redisS3InsecureSkipEnv = "S3_INSECURE_SKIP_VERIFY" +) + +// redisS3Trust mirrors how the default strategy script verifies the S3 +// endpoint: it trusts ca.crt, then tls.crt, from the s3-ca volume, and only +// when neither exists does it honour S3_INSECURE_SKIP_VERIFY=true. Both are +// read from the pod spec the backup Job ran with, but the Secret is read +// now, so an s3-ca volume that yields no CA now - a Secret that is gone or +// lacks the keys, or a source other than a Secret - is refused: the Job may +// have verified against a CA that is gone, and neither the system roots nor +// skip-verify would trust as narrowly. The default strategy mounts s3-ca +// only when the admin configured an endpoint CA. +func (r *BackupJobReconciler) redisS3Trust(ctx context.Context, namespace string, podSpec *corev1.PodSpec) ([]byte, bool, error) { + if podSpec == nil { + return nil, false, nil + } + caUnknown := false + for _, v := range podSpec.Volumes { + if v.Name != redisS3CAVolume { + continue + } + if v.Secret == nil { + caUnknown = true + break + } + secret := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: v.Secret.SecretName}, secret) + if apierrors.IsNotFound(err) { + caUnknown = true + break + } + if err != nil { + return nil, false, err + } + for _, file := range []string{"ca.crt", "tls.crt"} { + if pem := secret.Data[secretKeyForFile(v.Secret.Items, file)]; len(pem) > 0 { + return pem, false, nil + } + } + caUnknown = true + } + if caUnknown { + return nil, false, fmt.Errorf("the %s volume yields no CA now", redisS3CAVolume) + } + return nil, redisSkipsVerify(podSpec.Containers), nil +} + +// redisSkipsVerify reports whether every container sets +// S3_INSECURE_SKIP_VERIFY=true: the probe cannot tell which container +// uploaded, so one that verified is enough to keep verification on. When a +// container lists the variable twice, the last entry wins, as in the +// kubelet. +func redisSkipsVerify(containers []corev1.Container) bool { + for _, c := range containers { + value := "" + for _, e := range c.Env { + if e.Name == redisS3InsecureSkipEnv { + value = e.Value + } + } + if value != "true" { + return false + } + } + return len(containers) > 0 +} + +// secretKeyForFile returns the Secret key a secret volume projects to file: +// the key itself when the volume lists no items. +func secretKeyForFile(items []corev1.KeyToPath, file string) string { + if len(items) == 0 { + return file + } + for _, it := range items { + if it.Path == file { + return it.Key + } + } + return "" +} diff --git a/internal/backupcontroller/redis_artifact_size_test.go b/internal/backupcontroller/redis_artifact_size_test.go new file mode 100644 index 000000000..47d357c4e --- /dev/null +++ b/internal/backupcontroller/redis_artifact_size_test.go @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/pem" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +const redisTestKey = "tenant/cache/bj-1.rdb" + +func projectedCredsSecret(data map[string]string) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "cozy-backups-creds"}, + Data: map[string][]byte{}, + } + for k, v := range map[string]string{ + "endpoint": "s3.example.org", + "bucketName": "cozy-backups", + "region": "us-east-1", + "AWS_ACCESS_KEY_ID": "AKID", + "AWS_SECRET_ACCESS_KEY": "secret", + } { + s.Data[k] = []byte(v) + } + for k, v := range data { + if v == "" { + delete(s.Data, k) + continue + } + s.Data[k] = []byte(v) + } + return s +} + +func redisReconciler(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler { + r := reconcilerWithLister(t, l, objs...) + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + return r +} + +func TestRedisArtifact(t *testing.T) { + t.Run("stats the snapshot with the projected credentials", func(t *testing.T) { + l := &fakeS3Lister{statSize: 2048} + r := redisReconciler(t, l, projectedCredsSecret(nil)) + got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil) + if got == nil || got.URI != "s3://cozy-backups/"+redisTestKey || got.SizeBytes != 2048 { + t.Fatalf("artifact = %#v", got) + } + if l.bucket != "cozy-backups" || l.statKey != redisTestKey { + t.Errorf("stat %q/%q, want cozy-backups/%q", l.bucket, l.statKey, redisTestKey) + } + if l.conn.Endpoint != "s3.example.org" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.Region != "us-east-1" { + t.Errorf("connection = %+v", l.conn) + } + }) + + unset := map[string]struct { + lister *fakeS3Lister + objs []client.Object + noTarget bool + }{ + "missing object": {lister: &fakeS3Lister{statErr: errors.New("NoSuchKey")}, objs: []client.Object{projectedCredsSecret(nil)}}, + "missing Secret": {lister: &fakeS3Lister{statSize: 1}}, + "missing bucket": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"bucketName": ""})}}, + "missing keys": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"AWS_SECRET_ACCESS_KEY": ""})}}, + "no target name": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(nil)}, noTarget: true}, + "missing endpoint": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"endpoint": ""})}}, + "missing access key": {lister: &fakeS3Lister{statSize: 1}, objs: []client.Object{projectedCredsSecret(map[string]string{"AWS_ACCESS_KEY_ID": ""})}}, + } + for name, tc := range unset { + t.Run(name+" leaves the artifact unset", func(t *testing.T) { + r := redisReconciler(t, tc.lister, tc.objs...) + if tc.noTarget { + r.CredentialsConfig.TargetSecretName = "" + } + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil); got != nil { + t.Errorf("artifact = %#v, want nil", got) + } + }) + } +} + +// TestRedisArtifact_MinioClient runs the real minio client against a TLS +// stub with a self-signed certificate that answers the HEAD request. It +// covers, in one pass, the trust settings the backup Job used, request +// signing and Content-Length parsing. +func TestRedisArtifact_MinioClient(t *testing.T) { + // minio-go retries a TLS failure with backoff; a refusal needs no more + // than the first attempt. + saved := artifactSizeProbeTimeout + artifactSizeProbeTimeout = time.Second + defer func() { artifactSizeProbeTimeout = saved }() + var gotMethod, gotPath, gotAuth string + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotMethod, gotPath, gotAuth = req.Method, req.URL.Path, req.Header.Get("Authorization") + w.Header().Set("Content-Length", "4096") + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + + creds := projectedCredsSecret(map[string]string{"endpoint": strings.TrimPrefix(srv.URL, "https://")}) + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": caPEM}, + } + c := newMariaDBStrategyTestClient(t, creds, ca) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, &corev1.PodSpec{}); got != nil { + t.Errorf("without the Job's trust settings: artifact = %#v, want nil (untrusted certificate)", got) + } + + got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "")) + if got == nil || got.SizeBytes != 4096 { + t.Fatalf("with the s3-ca volume: artifact = %#v, want SizeBytes 4096", got) + } + if gotMethod != http.MethodHead || gotPath != "/cozy-backups/"+redisTestKey { + t.Errorf("request %s %s, want HEAD /cozy-backups/%s", gotMethod, gotPath, redisTestKey) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } + + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("", "true")); got == nil || got.SizeBytes != 4096 { + t.Errorf("with S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want SizeBytes 4096", got) + } + + ca.Data["ca.crt"] = unrelatedCAPEM(t) + if err := c.Update(context.Background(), ca); err != nil { + t.Fatalf("update CA Secret: %v", err) + } + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "true")); got != nil { + t.Errorf("with a CA that did not sign the endpoint and S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want nil", got) + } +} + +// redisPodSpec builds the parts of the backup Job's pod spec that +// redisS3Trust reads: the s3-ca volume when caSecret is set, and the +// S3_INSECURE_SKIP_VERIFY variable. +func redisPodSpec(caSecret, skipVerify string) *corev1.PodSpec { + spec := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: "backup", + Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: skipVerify}}, + }}} + if caSecret != "" { + spec.Volumes = []corev1.Volume{{ + Name: "s3-ca", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: caSecret}}, + }} + } + return spec +} + +func withVolume(spec *corev1.PodSpec, name, secret string) *corev1.PodSpec { + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: secret}}, + }) + return spec +} + +func configMapS3CA(spec *corev1.PodSpec) *corev1.PodSpec { + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "s3-ca", + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "s3-endpoint-ca"}}}, + }) + return spec +} + +// nextEvent returns the next recorded event, or "" when there is none. +func nextEvent(rec *record.FakeRecorder) string { + select { + case ev := <-rec.Events: + return ev + default: + return "" + } +} + +func TestRedisS3Trust(t *testing.T) { + caSecret := func(data map[string]string) *corev1.Secret { + s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, Data: map[string][]byte{}} + for k, v := range data { + s.Data[k] = []byte(v) + } + return s + } + cases := map[string]struct { + spec *corev1.PodSpec + objs []client.Object + wantCA string + wantSkip bool + }{ + "ca.crt wins over tls.crt and skip-verify": { + spec: redisPodSpec("s3-endpoint-ca", "true"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA", "tls.crt": "TLS"})}, + wantCA: "CA", + }, + "tls.crt when there is no ca.crt": { + spec: redisPodSpec("s3-endpoint-ca", ""), + objs: []client.Object{caSecret(map[string]string{"tls.crt": "TLS"})}, + wantCA: "TLS", + }, + "volume items map the file to another key": { + spec: func() *corev1.PodSpec { + s := redisPodSpec("s3-endpoint-ca", "") + s.Volumes[0].Secret.Items = []corev1.KeyToPath{{Key: "bundle.pem", Path: "ca.crt"}} + return s + }(), + objs: []client.Object{caSecret(map[string]string{"bundle.pem": "MAPPED", "ca.crt": "UNMAPPED"})}, + wantCA: "MAPPED", + }, + "skip-verify without a CA volume": { + spec: redisPodSpec("", "true"), + wantSkip: true, + }, + "skip-verify only when it is exactly true": { + spec: redisPodSpec("", "false"), + }, + "no pod spec": {}, + "the Redis CA volume is not the S3 CA": { + spec: withVolume(redisPodSpec("", "true"), "redis-ca", "redis-tls"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}}, + wantSkip: true, + }, + "the S3 CA next to the Redis CA": { + spec: withVolume(withVolume(&corev1.PodSpec{Containers: redisPodSpec("", "").Containers}, "redis-ca", "redis-tls"), "s3-ca", "s3-endpoint-ca"), + objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}}, + wantCA: "CA", + }, + "another variable set to true is not skip-verify": { + spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{{Name: "REDIS_TLS", Value: "true"}}}}}, + }, + "skip-verify only when every container skips": { + spec: &corev1.PodSpec{Containers: []corev1.Container{ + {Name: "upload", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"}}}, + {Name: "sidecar", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"}}}, + }}, + }, + "the last entry of a repeated variable wins": { + spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{ + {Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"}, + {Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"}, + }}}}, + }, + } + // Each s3-ca volume that yields no CA now is refused, with or without + // skip-verify: the Job may have verified against a CA that is gone. + refused := map[string]struct { + spec func(skip string) *corev1.PodSpec + objs []client.Object + }{ + "a CA Secret that is gone": {spec: func(skip string) *corev1.PodSpec { return redisPodSpec("s3-endpoint-ca", skip) }}, + "a CA Secret without CA keys": { + spec: func(skip string) *corev1.PodSpec { return redisPodSpec("s3-endpoint-ca", skip) }, + objs: []client.Object{caSecret(map[string]string{"other": "x"})}, + }, + "an s3-ca volume of another source": {spec: func(skip string) *corev1.PodSpec { return configMapS3CA(redisPodSpec("", skip)) }}, + "a CA Secret whose keys the volume does not project": { + spec: func(skip string) *corev1.PodSpec { + s := redisPodSpec("s3-endpoint-ca", skip) + s.Volumes[0].Secret.Items = []corev1.KeyToPath{{Key: "bundle.pem", Path: "ca.crt"}} + return s + }, + objs: []client.Object{caSecret(map[string]string{"tls.crt": "UNPROJECTED"})}, + }, + } + for name, tc := range refused { + for _, skip := range []string{"", "true"} { + t.Run(name+" is refused, skip="+skip, func(t *testing.T) { + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, tc.objs...)} + ca, gotSkip, err := r.redisS3Trust(context.Background(), "tenant", tc.spec(skip)) + if err == nil || gotSkip || ca != nil { + t.Errorf("trust = (%q, %v, %v), want an error, no CA and no skip-verify", ca, gotSkip, err) + } + }) + } + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := newMariaDBStrategyTestClient(t, tc.objs...) + r := &BackupJobReconciler{Client: c} + ca, skip, err := r.redisS3Trust(context.Background(), "tenant", tc.spec) + if err != nil { + t.Fatalf("redisS3Trust: %v", err) + } + if string(ca) != tc.wantCA || skip != tc.wantSkip { + t.Errorf("trust = (%q, %v), want (%q, %v)", ca, skip, tc.wantCA, tc.wantSkip) + } + }) + } +} + +func TestCreateRedisBackupArtifact_Artifact(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: "Redis", Name: "cache"}, + }, + } + + t.Run("records the snapshot location and size", func(t *testing.T) { + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 || got.Status.Artifact.URI != "s3://cozy-backups/"+redisTestKey { + t.Fatalf("Artifact = %#v", got.Status.Artifact) + } + }) + + t.Run("an unmeasured artifact is reported on the BackupJob without the raw error", func(t *testing.T) { + l := &fakeS3Lister{statErr: errors.New("Get https://s3.internal.example/cozy-backups: NoSuchKey")} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Status.Artifact != nil { + t.Fatalf("createRedisBackupArtifact = %#v, %v; want a Backup without an artifact", got, err) + } + select { + case ev := <-rec.Events: + if !strings.HasPrefix(ev, "Normal ArtifactNotMeasured ") || !strings.Contains(ev, "could not be found or read") { + t.Errorf("event = %q, want Normal ArtifactNotMeasured with the reason", ev) + } + if strings.Contains(ev, "s3.internal.example") { + t.Errorf("event = %q leaks the raw S3 error into the application namespace", ev) + } + default: + t.Error("no event on the BackupJob for an artifact left unset") + } + }) + + t.Run("a measured artifact raises no event", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statSize: 1}, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if len(rec.Events) != 0 { + t.Errorf("event %q for an artifact that was recorded", <-rec.Events) + } + }) + + t.Run("an existing Backup is returned without probing or creating", func(t *testing.T) { + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, existing, projectedCredsSecret(nil)) + creates := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + creates++ + return cl.Create(ctx, obj, opts...) + }, + }) + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times for a Backup that already exists", l.calls) + } + if creates != 0 { + t.Errorf("Create called %d times for a Backup that already exists", creates) + } + }) + + t.Run("an empty snapshot is recorded and warned about", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statSize: 0}, j, projectedCredsSecret(nil)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Status.Artifact == nil || got.Status.Artifact.URI != "s3://cozy-backups/"+redisTestKey { + t.Fatalf("createRedisBackupArtifact = %#v, %v; want the artifact recorded", got, err) + } + if ev := nextEvent(rec); !strings.HasPrefix(ev, "Warning EmptySnapshot ") { + t.Errorf("event = %q, want Warning EmptySnapshot", ev) + } + }) +} + +// TestReconcileRedis_ArtifactUsesTheJobTrust pins the wiring from the +// completed backup Job to the probe: the s3-ca volume of the pod the Job +// ran must reach the S3 connection. +func TestReconcileRedis_ArtifactUsesTheJobTrust(t *testing.T) { + app := newRedisApp("cache", "tenant-test") + strategy := newRedisStrategy("redis-strategy") + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newRedisAppRef("cache"), + BackupClassName: "cozy-default", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobNameForBackupJob(backupJob), + Namespace: backupJob.Namespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: backupJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace, + }, + }, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: *redisPodSpec("s3-endpoint-ca", "")}}, + Status: batchv1.JobStatus{ + CompletionTime: &now, + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + creds := projectedCredsSecret(nil) + creds.Namespace = "tenant-test" + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": []byte("CA")}, + } + resolved := newRedisResolved("redis-strategy", map[string]string{"bucketName": "redis-bucket"}) + + r, _ := newRedisTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob, creds, ca)) + l := &fakeS3Lister{statSize: 512} + r.CredentialsConfig.TargetSecretName = "cozy-backups-creds" + r.newS3ObjectLister = func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + } + ctx := context.Background() + + if _, err := r.reconcileRedis(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileRedis() error = %v", err) + } + if string(l.conn.CAPEM) != "CA" { + t.Errorf("connection CA = %q, want the s3-ca volume's ca.crt", l.conn.CAPEM) + } + + updated := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil { + t.Fatalf("get backupjob: %v", err) + } + if updated.Status.BackupRef == nil { + t.Fatalf("expected BackupRef set") + } + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: backupJob.Namespace, Name: updated.Status.BackupRef.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if created.Status.Artifact == nil || created.Status.Artifact.SizeBytes != 512 { + t.Errorf("Artifact = %#v, want SizeBytes 512", created.Status.Artifact) + } +} + +func TestCreateRedisBackupArtifact_RetryCheck(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: "Redis", Name: "cache"}, + }, + } + + t.Run("a failed existence check creates the Backup without probing", func(t *testing.T) { + l := &fakeS3Lister{statSize: 777} + r := redisReconciler(t, l, j, projectedCredsSecret(nil)) + r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t)) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil { + t.Fatalf("createRedisBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } + if ev := nextEvent(rec); !strings.HasPrefix(ev, "Normal ArtifactNotMeasured ") || !strings.Contains(ev, "could not be checked") { + t.Errorf("event = %q, want ArtifactNotMeasured saying the existence check failed", ev) + } + }) + + t.Run("no event when Create fails", func(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{statErr: errors.New("NoSuchKey")}, j, projectedCredsSecret(nil)) + r.Client = forbidBackupCreate(t, r.Client) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err == nil { + t.Fatal("createRedisBackupArtifact succeeded although Create was rejected") + } + if ev := nextEvent(rec); ev != "" { + t.Errorf("event %q although the BackupJob fails", ev) + } + }) + + t.Run("no event when Create collides with a Backup another pass made", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statErr: errors.New("NoSuchKey")}) + r.Client, r.APIReader = staleCache(t, store), notFoundOnce(t, store) + rec := record.NewFakeRecorder(4) + r.Recorder = rec + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); err != nil { + t.Fatalf("createRedisBackupArtifact: %v", err) + } + if ev := nextEvent(rec); ev != "" { + t.Errorf("event %q about a Backup this pass did not create", ev) + } + }) + + t.Run("a Create collision is resolved from the API server past the cache", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statSize: 777}) + r.Client, r.APIReader = staleCache(t, store), notFoundOnce(t, store) + got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil) + if err != nil || got == nil || got.Name != "bj-1" { + t.Fatalf("createRedisBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } + }) + + t.Run("an existing Backup that cannot be read back is retried", func(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}} + store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil)) + r := redisReconciler(t, &fakeS3Lister{statSize: 777}) + r.Client, r.APIReader = staleCache(t, store), failingReads(t, store) + if _, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil); !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } + }) +} + +func TestRedisArtifact_HangingEndpointIsGivenUp(t *testing.T) { + r := redisReconciler(t, &fakeS3Lister{}, projectedCredsSecret(nil)) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, nil); got != nil { + t.Errorf("artifact = %#v, want nil", got) + } + }) +} + +func TestRedisArtifact_TrustsOnlyTheCA(t *testing.T) { + l := &fakeS3Lister{statSize: 1} + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + r := redisReconciler(t, l, projectedCredsSecret(nil), ca) + if got, _ := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "")); got == nil { + t.Fatal("artifact = nil") + } + if !l.conn.CAOnly { + t.Error("the probe trusts the system roots on top of the CA, which curl --cacert does not") + } +} + +// TestReconcileRedis_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileRedis_UnreadableBackupIsRetried(t *testing.T) { + app := newRedisApp("cache", "tenant-test") + strategy := newRedisStrategy("redis-strategy") + now := metav1.Now() + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newRedisAppRef("cache"), + BackupClassName: "cozy-default", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobNameForBackupJob(backupJob), + Namespace: backupJob.Namespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: backupJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace, + }, + }, + Status: batchv1.JobStatus{ + CompletionTime: &now, + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: "test-bj"}} + resolved := newRedisResolved("redis-strategy", map[string]string{"bucketName": "redis-bucket"}) + + r, _ := newRedisTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob, existing)) + store := r.Client + r.Client, r.APIReader = staleCache(t, store), failingReads(t, store) + ctx := context.Background() + + if _, err := r.reconcileRedis(ctx, backupJob, resolved); !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileRedis err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(ctx, client.ObjectKeyFromObject(backupJob), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} diff --git a/internal/backupcontroller/redisstrategy_controller.go b/internal/backupcontroller/redisstrategy_controller.go index 7d287d385..4169e6cef 100644 --- a/internal/backupcontroller/redisstrategy_controller.go +++ b/internal/backupcontroller/redisstrategy_controller.go @@ -3,6 +3,7 @@ package backupcontroller import ( "context" + "errors" "fmt" "maps" "strings" @@ -366,9 +367,9 @@ func (r *BackupJobReconciler) reconcileRedis(ctx context.Context, j *backupsv1al if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createRedisBackupArtifact(ctx, j, resolved, objectKey) + artifact, err := r.createRedisBackupArtifact(ctx, j, resolved, objectKey, &batchJob.Spec.Template.Spec) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -471,7 +472,12 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig, objectKey string, + podSpec *corev1.PodSpec, ) (*backupsv1alpha1.Backup, error) { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { + return existing, nil + } driverMD := map[string]string{redisObjectMetaKey: objectKey} for k, v := range resolved.Parameters { driverMD[redisParamPrefix+k] = v @@ -492,6 +498,19 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( Phase: backupsv1alpha1.BackupPhaseReady, }, } + unmeasured := "whether the Backup already exists could not be checked" + if probe { + artifact, err := r.redisArtifact(ctx, j.Namespace, objectKey, podSpec) + unmeasured = "" + if err != nil { + getLogger(ctx).Info("leaving the Redis backup artifact unset", "backupjob", j.Name, "objectKey", objectKey, "error", err) + var probeErr *artifactProbeError + if errors.As(err, &probeErr) { + unmeasured = probeErr.reason + } + } + backup.Status.Artifact = artifact + } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef } @@ -500,11 +519,23 @@ func (r *BackupJobReconciler) createRedisBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } + // Only once the Backup exists: a terminal Create error fails the + // BackupJob, and a collision returns a Backup another pass measured. + if r.Recorder != nil { + switch { + case unmeasured != "": + r.Recorder.Eventf(j, corev1.EventTypeNormal, "ArtifactNotMeasured", "The backup succeeded, but its location and size were not recorded: %s", unmeasured) + case backup.Status.Artifact != nil && backup.Status.Artifact.SizeBytes == 0: + // sizeBytes is omitempty, so a 0 reads as "not measured" on the + // Backup; a valid RDB snapshot always carries a header. + r.Recorder.Event(j, corev1.EventTypeWarning, "EmptySnapshot", "The backup's RDB snapshot is empty (0 bytes); a valid snapshot is never empty, so this backup is likely broken.") + } + } return backup, nil } diff --git a/internal/observabilityproxy/auth.go b/internal/observabilityproxy/auth.go new file mode 100644 index 000000000..0baac558a --- /dev/null +++ b/internal/observabilityproxy/auth.go @@ -0,0 +1,172 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// ErrUnauthenticated means the token is missing or kube-apiserver rejected it. +var ErrUnauthenticated = errors.New("unauthenticated") + +// Authorizer asks kube-apiserver who a token belongs to and whether that +// identity may read a scope. Answers, including negative ones, are cached +// for the TTL; transport errors and reviews that failed to decide are not, +// so an outage is retried on the next request instead of locking a user +// out for the TTL. +type Authorizer struct { + client kubernetes.Interface + now func() time.Time + tokens *ttlCache[*authenticationv1.UserInfo] + decisions *ttlCache[bool] +} + +func NewAuthorizer(client kubernetes.Interface, ttl time.Duration) *Authorizer { + return &Authorizer{ + client: client, + now: time.Now, + tokens: newTTLCache[*authenticationv1.UserInfo](ttl), + decisions: newTTLCache[bool](ttl), + } +} + +func (a *Authorizer) Authenticate(ctx context.Context, token string) (*authenticationv1.UserInfo, error) { + if token == "" { + return nil, ErrUnauthenticated + } + sum := sha256.Sum256([]byte(token)) + key := hex.EncodeToString(sum[:]) + if u, ok := a.tokens.get(key, a.now()); ok { + if u == nil { + return nil, ErrUnauthenticated + } + return u, nil + } + tr, err := a.client.AuthenticationV1().TokenReviews().Create(ctx, + &authenticationv1.TokenReview{Spec: authenticationv1.TokenReviewSpec{Token: token}}, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("token review: %w", err) + } + if !tr.Status.Authenticated || tr.Status.User.Username == "" { + // status.error also marks an authenticator that failed to decide; + // a cached refusal would outlast its recovery. + if tr.Status.Error == "" { + a.tokens.set(key, nil, a.now()) + } + return nil, ErrUnauthenticated + } + u := tr.Status.User.DeepCopy() + a.tokens.set(key, u, a.now()) + return u, nil +} + +func (a *Authorizer) Authorize(ctx context.Context, user *authenticationv1.UserInfo, attrs authorizationv1.ResourceAttributes) (bool, error) { + extra := make(map[string]authorizationv1.ExtraValue, len(user.Extra)) + for k, v := range user.Extra { + extra[k] = authorizationv1.ExtraValue(v) + } + spec := authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &attrs, + User: user.Username, + UID: user.UID, + Groups: user.Groups, + Extra: extra, + } + raw, err := json.Marshal(spec) + if err != nil { + return false, err + } + key := string(raw) + if allowed, ok := a.decisions.get(key, a.now()); ok { + return allowed, nil + } + sar, err := a.client.AuthorizationV1().SubjectAccessReviews().Create(ctx, + &authorizationv1.SubjectAccessReview{Spec: spec}, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Errorf("subject access review: %w", err) + } + allowed := sar.Status.Allowed && !sar.Status.Denied + // An evaluation error without a decision denies this request only: it + // may be a webhook authorizer that is down, and RBAC reports one for + // any binding to a missing role, so it cannot be told from a refusal. + if allowed || sar.Status.Denied || sar.Status.EvaluationError == "" { + a.decisions.set(key, allowed, a.now()) + } + return allowed, nil +} + +// ttlCacheMaxEntries bounds memory against a flood of distinct tokens; once +// reached, expired entries are swept and, if that is not enough, the cache +// starts over, which costs only extra reviews. +const ttlCacheMaxEntries = 10000 + +type ttlEntry[V any] struct { + value V + expires time.Time +} + +type ttlCache[V any] struct { + mu sync.Mutex + ttl time.Duration + entries map[string]ttlEntry[V] +} + +func newTTLCache[V any](ttl time.Duration) *ttlCache[V] { + return &ttlCache[V]{ttl: ttl, entries: map[string]ttlEntry[V]{}} +} + +func (c *ttlCache[V]) get(key string, now time.Time) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok || !now.Before(e.expires) { + var zero V + return zero, false + } + return e.value, true +} + +func (c *ttlCache[V]) set(key string, v V, now time.Time) { + if c.ttl <= 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.entries) >= ttlCacheMaxEntries { + for k, e := range c.entries { + if !now.Before(e.expires) { + delete(c.entries, k) + } + } + if len(c.entries) >= ttlCacheMaxEntries { + c.entries = map[string]ttlEntry[V]{} + } + } + c.entries[key] = ttlEntry[V]{value: v, expires: now.Add(c.ttl)} +} diff --git a/internal/observabilityproxy/auth_test.go b/internal/observabilityproxy/auth_test.go new file mode 100644 index 000000000..fe940ce04 --- /dev/null +++ b/internal/observabilityproxy/auth_test.go @@ -0,0 +1,220 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// fakeKube answers TokenReviews for the tokens in users and allows a +// SubjectAccessReview when allow returns true; it counts both calls. +type fakeKube struct { + client *fake.Clientset + reviews atomic.Int32 + sars atomic.Int32 + lastSAR atomic.Pointer[authorizationv1.SubjectAccessReviewSpec] + reviewFn func() error + // reviewError and evaluationError, when set, are what the + // authenticator or the authorizer report as failing to decide. + reviewError atomic.Pointer[string] + evaluationError atomic.Pointer[string] +} + +func newFakeKube(users map[string]string, allow func(user string, attrs *authorizationv1.ResourceAttributes) bool) *fakeKube { + f := &fakeKube{client: fake.NewClientset()} + f.client.PrependReactor("create", "tokenreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + f.reviews.Add(1) + if f.reviewFn != nil { + if err := f.reviewFn(); err != nil { + return true, nil, err + } + } + tr := action.(k8stesting.CreateAction).GetObject().(*authenticationv1.TokenReview).DeepCopy() + if e := f.reviewError.Load(); e != nil { + tr.Status = authenticationv1.TokenReviewStatus{Error: *e} + } else if name, ok := users[tr.Spec.Token]; ok { + tr.Status = authenticationv1.TokenReviewStatus{Authenticated: true, User: authenticationv1.UserInfo{Username: name, Groups: []string{"tenant-a-view"}}} + } + return true, tr, nil + }) + f.client.PrependReactor("create", "subjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + f.sars.Add(1) + sar := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SubjectAccessReview).DeepCopy() + spec := sar.Spec + f.lastSAR.Store(&spec) + if e := f.evaluationError.Load(); e != nil { + sar.Status.EvaluationError = *e + return true, sar, nil + } + sar.Status.Allowed = allow(sar.Spec.User, sar.Spec.ResourceAttributes) + return true, sar, nil + }) + return f +} + +func allowNamespace(ns string) func(string, *authorizationv1.ResourceAttributes) bool { + return func(_ string, a *authorizationv1.ResourceAttributes) bool { return a != nil && a.Namespace == ns } +} + +func TestAuthenticate(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + + u, err := a.Authenticate(ctx, "good") + if err != nil || u.Username != "alice" { + t.Fatalf("Authenticate(good) = %+v, %v", u, err) + } + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + if n := kube.reviews.Load(); n != 1 { + t.Fatalf("TokenReviews = %d, want 1 (second call cached)", n) + } + if _, err := a.Authenticate(ctx, "bad"); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("Authenticate(bad) err = %v, want ErrUnauthenticated", err) + } + if _, err := a.Authenticate(ctx, ""); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("Authenticate(\"\") err = %v, want ErrUnauthenticated", err) + } +} + +func TestAuthenticateCacheExpires(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + now := time.Unix(1000, 0) + a.now = func() time.Time { return now } + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + now = now.Add(11 * time.Second) + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatal(err) + } + if n := kube.reviews.Load(); n != 2 { + t.Fatalf("TokenReviews = %d, want 2 after TTL", n) + } +} + +func TestAuthenticateErrorNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + kube.reviewFn = func() error { return errors.New("apiserver down") } + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); err == nil || errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want a transport error", err) + } + kube.reviewFn = nil + if _, err := a.Authenticate(ctx, "good"); err != nil { + t.Fatalf("err = %v after recovery", err) + } +} + +func TestAuthorize(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + user := &authenticationv1.UserInfo{Username: "alice", UID: "u1", Groups: []string{"g"}, Extra: map[string]authenticationv1.ExtraValue{"scope": {"x"}}} + + allowed := Route{Kind: NamespaceLogs, Namespace: "tenant-a"} + denied := Route{Kind: NamespaceLogs, Namespace: "tenant-b"} + + ok, err := a.Authorize(ctx, user, allowed.ResourceAttributes()) + if err != nil || !ok { + t.Fatalf("allow: %v, %v", ok, err) + } + spec := kube.lastSAR.Load() + if spec.User != "alice" || spec.UID != "u1" || len(spec.Groups) != 1 || spec.Extra["scope"][0] != "x" { + t.Fatalf("SAR identity not forwarded: %+v", spec) + } + if ra := spec.ResourceAttributes; ra.Verb != "list" || ra.Group != "cozystack.io" || ra.Resource != "logs" || ra.Namespace != "tenant-a" { + t.Fatalf("SAR attributes = %+v", ra) + } + + ok, err = a.Authorize(ctx, user, denied.ResourceAttributes()) + if err != nil || ok { + t.Fatalf("deny: %v, %v", ok, err) + } + + for range 3 { + if ok, _ := a.Authorize(ctx, user, allowed.ResourceAttributes()); !ok { + t.Fatal("cached allow flipped") + } + if ok, _ := a.Authorize(ctx, user, denied.ResourceAttributes()); ok { + t.Fatal("cached deny flipped") + } + } + if n := kube.sars.Load(); n != 2 { + t.Fatalf("SubjectAccessReviews = %d, want 2 (rest cached)", n) + } + + other := &authenticationv1.UserInfo{Username: "alice", UID: "u1", Groups: []string{"g", "tenant-b-view"}} + if _, err := a.Authorize(ctx, other, denied.ResourceAttributes()); err != nil { + t.Fatal(err) + } + if n := kube.sars.Load(); n != 3 { + t.Fatalf("a different group set must not hit another identity's cache entry, SARs = %d", n) + } +} + +// TestAuthenticateFailedReviewNotCached: a TokenReview the authenticator +// could not decide carries status.error; caching it would refuse a valid +// token for the TTL after the authenticator recovers. +func TestAuthenticateFailedReviewNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + outage := "webhook authenticator unreachable" + kube.reviewError.Store(&outage) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + if _, err := a.Authenticate(ctx, "good"); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + kube.reviewError.Store(nil) + if u, err := a.Authenticate(ctx, "good"); err != nil || u.Username != "alice" { + t.Fatalf("after recovery: %+v, %v", u, err) + } +} + +// TestAuthorizeEvaluationErrorNotCached: a SubjectAccessReview with an +// evaluation error and no decision is a denial for this request only. +func TestAuthorizeEvaluationErrorNotCached(t *testing.T) { + kube := newFakeKube(map[string]string{"good": "alice"}, allowNamespace("tenant-a")) + failure := "webhook authorizer unreachable" + kube.evaluationError.Store(&failure) + a := NewAuthorizer(kube.client, 10*time.Second) + ctx := context.Background() + user := &authenticationv1.UserInfo{Username: "alice"} + attrs := Route{Kind: NamespaceLogs, Namespace: "tenant-a"}.ResourceAttributes() + if ok, err := a.Authorize(ctx, user, attrs); err != nil || ok { + t.Fatalf("undecided review = %v, %v; want a denial", ok, err) + } + kube.evaluationError.Store(nil) + if ok, err := a.Authorize(ctx, user, attrs); err != nil || !ok { + t.Fatalf("after recovery: %v, %v", ok, err) + } +} diff --git a/internal/observabilityproxy/limits.go b/internal/observabilityproxy/limits.go new file mode 100644 index 000000000..06cc41d41 --- /dev/null +++ b/internal/observabilityproxy/limits.go @@ -0,0 +1,166 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "sync" + "time" +) + +// LimiterConfig holds the per-user and total caps. A zero cap disables it. +type LimiterConfig struct { + MaxConcurrentPerUser int + RatePerUser float64 + RateBurst int + MaxTailsPerUser int + MaxTails int +} + +// limiter keeps open tails apart from the request concurrency budget: a +// tail holds its slot for minutes, and counting it there would starve the +// same user's dashboard panels. +type limiter struct { + cfg LimiterConfig + now func() time.Time + mu sync.Mutex + active map[string]int + tails map[string]int + total int + buckets map[string]*bucket +} + +type bucket struct { + tokens float64 + last time.Time +} + +func newLimiter(cfg LimiterConfig) *limiter { + return &limiter{ + cfg: cfg, + now: time.Now, + active: map[string]int{}, + tails: map[string]int{}, + buckets: map[string]*bucket{}, + } +} + +// allowRate takes a token from the user's bucket; when it is empty, it +// returns how long until the next token. +func (l *limiter) allowRate(user string) (time.Duration, bool) { + if l.cfg.RatePerUser <= 0 { + return 0, true + } + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + burst := float64(max(l.cfg.RateBurst, 1)) + b, ok := l.buckets[user] + if !ok { + if len(l.buckets) >= ttlCacheMaxEntries { + // Every bucket still refilling means that many users within one + // refill period; a newcomer waits for the first to drop out + // rather than grow the map past its bound. + if soonest := l.sweepBuckets(now, burst); len(l.buckets) >= ttlCacheMaxEntries { + return soonest, false + } + } + b = &bucket{tokens: burst, last: now} + l.buckets[user] = b + } + b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*l.cfg.RatePerUser) + b.last = now + if b.tokens < 1 { + wait, ok := scaleDuration((1-b.tokens)/l.cfg.RatePerUser, time.Second) + if !ok { + wait = math.MaxInt64 + } + return wait, false + } + b.tokens-- + return 0, true +} + +// sweepBuckets drops buckets that have refilled completely; recreating one +// later starts it full, which is the same state. It returns how long until +// the first remaining bucket is full. +func (l *limiter) sweepBuckets(now time.Time, burst float64) time.Duration { + soonest := time.Duration(math.MaxInt64) + for user, b := range l.buckets { + missing := burst - (b.tokens + now.Sub(b.last).Seconds()*l.cfg.RatePerUser) + if missing <= 0 { + delete(l.buckets, user) + continue + } + if wait, ok := scaleDuration(missing/l.cfg.RatePerUser, time.Second); ok { + soonest = min(soonest, max(wait, time.Nanosecond)) + } + } + return soonest +} + +func (l *limiter) acquire(user string) (func(), bool) { + l.mu.Lock() + defer l.mu.Unlock() + if l.cfg.MaxConcurrentPerUser > 0 && l.active[user] >= l.cfg.MaxConcurrentPerUser { + return nil, false + } + l.active[user]++ + return l.releaser(func() { decrement(l.active, user) }), true +} + +func (l *limiter) acquireTail(user string) (func(), bool) { + l.mu.Lock() + defer l.mu.Unlock() + if l.cfg.MaxTailsPerUser > 0 && l.tails[user] >= l.cfg.MaxTailsPerUser { + return nil, false + } + if l.cfg.MaxTails > 0 && l.total >= l.cfg.MaxTails { + return nil, false + } + l.tails[user]++ + l.total++ + return l.releaser(func() { + decrement(l.tails, user) + l.total-- + }), true +} + +func (l *limiter) openTails() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.total +} + +func (l *limiter) releaser(release func()) func() { + var once sync.Once + return func() { + once.Do(func() { + l.mu.Lock() + defer l.mu.Unlock() + release() + }) + } +} + +func decrement(m map[string]int, user string) { + if m[user] <= 1 { + delete(m, user) + return + } + m[user]-- +} diff --git a/internal/observabilityproxy/limits_test.go b/internal/observabilityproxy/limits_test.go new file mode 100644 index 000000000..d7f4c76f4 --- /dev/null +++ b/internal/observabilityproxy/limits_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "testing" + "time" +) + +func TestLimiterConcurrency(t *testing.T) { + l := newLimiter(LimiterConfig{MaxConcurrentPerUser: 2}) + r1, ok1 := l.acquire("alice") + _, ok2 := l.acquire("alice") + if !ok1 || !ok2 { + t.Fatal("first two requests refused") + } + if _, ok := l.acquire("alice"); ok { + t.Fatal("third concurrent request admitted") + } + if _, ok := l.acquire("bob"); !ok { + t.Fatal("another user is limited by alice's requests") + } + r1() + r1() + if _, ok := l.acquire("alice"); !ok { + t.Fatal("slot not returned on release") + } + if _, ok := l.acquire("alice"); ok { + t.Fatal("double release freed a second slot") + } +} + +func TestLimiterTails(t *testing.T) { + l := newLimiter(LimiterConfig{MaxConcurrentPerUser: 1, MaxTailsPerUser: 2, MaxTails: 3}) + if _, ok := l.acquire("alice"); !ok { + t.Fatal("request refused") + } + a1, ok1 := l.acquireTail("alice") + _, ok2 := l.acquireTail("alice") + if !ok1 || !ok2 { + t.Fatal("tails share the request concurrency budget") + } + if _, ok := l.acquireTail("alice"); ok { + t.Fatal("per-user tail cap not enforced") + } + if _, ok := l.acquireTail("bob"); !ok { + t.Fatal("bob refused below the total cap") + } + if _, ok := l.acquireTail("carol"); ok { + t.Fatal("total tail cap not enforced") + } + if got := l.openTails(); got != 3 { + t.Fatalf("openTails = %d, want 3", got) + } + a1() + if _, ok := l.acquireTail("carol"); !ok { + t.Fatal("total slot not returned on release") + } +} + +func TestLimiterRate(t *testing.T) { + now := time.Unix(1000, 0) + l := newLimiter(LimiterConfig{RatePerUser: 2, RateBurst: 2}) + l.now = func() time.Time { return now } + for i := range 2 { + if !allowed(l, "alice") { + t.Fatalf("burst request %d refused", i+1) + } + } + if wait, ok := l.allowRate("alice"); ok || wait != 500*time.Millisecond { + t.Fatalf("request over the burst: admitted %v, wait %s, want refused with 500ms", ok, wait) + } + if !allowed(l, "bob") { + t.Fatal("bob limited by alice's rate") + } + now = now.Add(500 * time.Millisecond) + if !allowed(l, "alice") { + t.Fatal("token not refilled after 1/rate") + } + now = now.Add(100 * time.Millisecond) + if wait, ok := l.allowRate("alice"); ok || wait != 400*time.Millisecond { + t.Fatalf("partly refilled bucket: admitted %v, wait %s, want refused with 400ms", ok, wait) + } +} + +func TestLimiterRateDisabled(t *testing.T) { + l := newLimiter(LimiterConfig{}) + for range 100 { + if !allowed(l, "alice") { + t.Fatal("rate limit applied while disabled") + } + } +} + +func allowed(l *limiter, user string) bool { + _, ok := l.allowRate(user) + return ok +} + +func TestLimiterRateWaitAtTinyRate(t *testing.T) { + l := newLimiter(LimiterConfig{RatePerUser: 1e-12, RateBurst: 1}) + allowed(l, "alice") + if wait, ok := l.allowRate("alice"); ok || wait <= 0 { + t.Fatalf("empty bucket at a tiny rate: admitted %v, wait %s, want refused with a positive wait", ok, wait) + } +} + +// TestLimiterRateBucketsBounded fills the bucket map with users that are +// all still refilling: a new user must be refused rather than grow it, and +// admitted once a bucket can be dropped. +func TestLimiterRateBucketsBounded(t *testing.T) { + now := time.Unix(1000, 0) + l := newLimiter(LimiterConfig{RatePerUser: 2, RateBurst: 2}) + l.now = func() time.Time { return now } + for i := range ttlCacheMaxEntries { + if !allowed(l, fmt.Sprintf("user-%d", i)) { + t.Fatalf("user-%d refused", i) + } + now = now.Add(time.Microsecond) + } + wait, ok := l.allowRate("newcomer") + if ok || wait <= 0 { + t.Fatalf("newcomer on a full map: admitted %v, wait %s; want refused with a positive wait", ok, wait) + } + if n := len(l.buckets); n > ttlCacheMaxEntries { + t.Fatalf("bucket map grew to %d, above %d", n, ttlCacheMaxEntries) + } + now = now.Add(wait) + if !allowed(l, "newcomer") { + t.Fatalf("newcomer still refused after the advertised wait %s", wait) + } +} diff --git a/internal/observabilityproxy/logsparams.go b/internal/observabilityproxy/logsparams.go new file mode 100644 index 000000000..ec6043c6f --- /dev/null +++ b/internal/observabilityproxy/logsparams.go @@ -0,0 +1,426 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "math" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +// LogsLimits bound what one logs request may make vlselect do. +type LogsLimits struct { + DefaultRange time.Duration + MaxRange time.Duration + MaxBuckets int + DefaultLimit int + MaxLimit int + MaxTailResume time.Duration +} + +// tailOffset is vlselect's default tail offset. The proxy never forwards a +// client offset, so the resume arithmetic can rely on this value. +const tailOffset = 5 * time.Second + +const maxHitsFields = 10 + +var logsEndpoints = map[string]bool{ + "query": true, "hits": true, "stats_query_range": true, "tail": true, + "field_values": true, "stream_field_values": true, "facets": true, +} + +// BuildLogsParams rebuilds the upstream parameters from scratch: only the +// allowlisted parameters of the endpoint are read from the client, and the +// namespace scope is added as the only extra_stream_filters value. +// VictoriaLogs ANDs extra_stream_filters into the query and every subquery +// (union, join, in()), which is what lets the query itself pass through +// unparsed. +func BuildLogsParams(endpoint string, in url.Values, namespace string, now time.Time, lim LogsLimits) (url.Values, *HTTPError) { + if !logsEndpoints[endpoint] { + return nil, errorf(http.StatusNotFound, "not found") + } + query := in.Get("query") + if query == "" { + query = "*" + } + // ignore_global_time_filter lifts the time window this function + // enforces, and on tail rescans the whole retention every tick. A + // substring match for it is bypassable (VictoriaLogs unquotes a quoted + // option name and joins words split by a comment), and no option is + // needed by a tenant, so any options(...) block is refused. + if hasOptionsBlock(query) { + return nil, errorf(http.StatusBadRequest, "query options are not allowed") + } + out := url.Values{} + out.Set("query", query) + out.Set("extra_stream_filters", "{kubernetes_namespace_name="+strconv.Quote(namespace)+"}") + + if endpoint == "tail" { + return out, tailParams(in, out, now, lim) + } + + start, end, herr := window(in, now, lim) + if herr != nil { + return nil, herr + } + out.Set("start", start.Format(time.RFC3339Nano)) + out.Set("end", end.Format(time.RFC3339Nano)) + + switch endpoint { + case "query": + herr = setLimit(out, in, "limit", lim.DefaultLimit, lim.MaxLimit) + case "hits": + if herr = setStep(out, in, start, end, lim); herr != nil { + return nil, herr + } + fields := in["field"] + if len(fields) > maxHitsFields { + return nil, errorf(http.StatusBadRequest, "at most %d field parameters are allowed", maxHitsFields) + } + for _, f := range fields { + if f != "" { + out.Add("field", f) + } + } + // vlselect returns every group without fields_limit or with 0, so + // the cap is sent either way. + herr = setLimit(out, in, "fields_limit", lim.MaxLimit, lim.MaxLimit) + case "stats_query_range": + herr = setStep(out, in, start, end, lim) + case "field_values", "stream_field_values": + if in.Get("field") == "" { + return nil, errorf(http.StatusBadRequest, "field is required") + } + out.Set("field", in.Get("field")) + if f := in.Get("filter"); f != "" { + out.Set("filter", f) + } + herr = setLimit(out, in, "limit", lim.DefaultLimit, lim.MaxLimit) + case "facets": + for _, k := range []string{"limit", "max_values_per_field", "max_value_len"} { + if k == "limit" || in.Has(k) { + if herr = setLimit(out, in, k, lim.DefaultLimit, lim.MaxLimit); herr != nil { + return nil, herr + } + } + } + if v := in.Get("keep_const_fields"); v == "1" || v == "true" { + out.Set("keep_const_fields", "1") + } + } + if herr != nil { + return nil, herr + } + return out, nil +} + +// window derives a finite [start, end]: VictoriaLogs otherwise starts at +// the oldest stored entry and scans the whole retention. +func window(in url.Values, now time.Time, lim LogsLimits) (time.Time, time.Time, *HTTPError) { + end := now + if v := in.Get("end"); v != "" { + t, err := parseTime(v, now) + if err != nil { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "invalid end: %v", err) + } + end = t + } + start := end.Add(-lim.DefaultRange) + if v := in.Get("start"); v != "" { + t, err := parseTime(v, now) + if err != nil { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "invalid start: %v", err) + } + start = t + } + if start.After(end) { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end") + } + if end.Sub(start) > lim.MaxRange { + start = end.Add(-lim.MaxRange) + } + if herr := sendableTime("end", end); herr != nil { + return time.Time{}, time.Time{}, herr + } + if herr := sendableTime("start", start); herr != nil { + return time.Time{}, time.Time{}, herr + } + return start.UTC(), end.UTC(), nil +} + +// setStep raises step so that hits and stats_query_range, which return one +// bucket per step, return at most MaxBuckets buckets, and caps it so the +// window cannot grow through it: vlselect widens [start, end] outward to +// step boundaries (alignStartEndToStep), by up to one step on each side, +// so a 100y step would otherwise scan the whole retention. The window is +// shrunk from the start until the widened span fits MaxRange. +func setStep(out, in url.Values, start, end time.Time, lim LogsLimits) *HTTPError { + span := end.Sub(start) + // vlselect returns a zero-filled bucket for every step boundary from + // the aligned start through the aligned end, one more than span/step + // whenever start is off a boundary, so one bucket is held back. + spans := time.Duration(lim.MaxBuckets - 1) + minStep := max((span+spans-1)/spans, time.Second) + step := minStep + if v := in.Get("step"); v != "" { + d, err := parseDuration(v) + if err != nil || d <= 0 { + return errorf(http.StatusBadRequest, "invalid step %q", v) + } + step = max(min(d, span, lim.MaxRange/3), minStep) + } + // vlselect gets whole milliseconds; rounding down would add a bucket. + if r := step % time.Millisecond; r != 0 { + step += time.Millisecond - r + } + if span+2*step > lim.MaxRange { + start = end.Add(-max(lim.MaxRange-2*step, 0)) + } + out.Set("step", formatDuration(step)) + out.Set("start", start.Format(time.RFC3339Nano)) + return nil +} + +// setLimit refuses an empty value rather than read it as the default: a +// cap the client sent but left blank is malformed, the same for every cap, +// where reading it as the default would hide a client bug. +func setLimit(out, in url.Values, key string, def, maxv int) *HTTPError { + n := def + if in.Has(key) && in.Get(key) == "" { + return errorf(http.StatusBadRequest, "invalid %s \"\"", key) + } + if v := in.Get(key); v != "" { + parsed, err := strconv.Atoi(v) + if err != nil || parsed <= 0 { + return errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + n = parsed + } + out.Set(key, strconv.Itoa(min(n, maxv))) + return nil +} + +// tailParams turns a resume point into a capped start_offset. A reconnect +// older than the cap resumes at the cap; the gap needs a historical query. +func tailParams(in, out url.Values, now time.Time, lim LogsLimits) *HTTPError { + var start, end time.Time + for _, k := range []string{"start", "end"} { + v := in.Get(k) + if v == "" { + continue + } + t, err := parseTime(v, now) + if err != nil { + return errorf(http.StatusBadRequest, "invalid %s: %v", k, err) + } + if herr := sendableTime(k, t); herr != nil { + return herr + } + out.Set(k, t.UTC().Format(time.RFC3339Nano)) + if k == "start" { + start = t + } else { + end = t + } + } + if !start.IsZero() && !end.IsZero() && start.After(end) { + return errorf(http.StatusBadRequest, "start is after end") + } + + since, offset := in.Get("since"), in.Get("start_offset") + var d time.Duration + switch { + case since != "" && offset != "": + return errorf(http.StatusBadRequest, "since and start_offset are mutually exclusive") + case since != "": + t, err := parseTime(since, now) + if err != nil { + return errorf(http.StatusBadRequest, "invalid since: %v", err) + } + d = now.Add(-tailOffset).Sub(t) + case offset != "": + parsed, err := parseDuration(offset) + if err != nil { + return errorf(http.StatusBadRequest, "invalid start_offset %q", offset) + } + d = parsed + default: + return nil + } + out.Set("start_offset", formatDuration(min(max(d, 0), lim.MaxTailResume))) + return nil +} + +// formatDuration writes whole milliseconds, a form every VictoriaLogs +// duration parser accepts. +func formatDuration(d time.Duration) string { + return fmt.Sprintf("%dms", d.Milliseconds()) +} + +var durationToken = regexp.MustCompile(`^(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) + +var durationUnits = map[string]time.Duration{ + "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, + "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, +} + +// parseDuration accepts the LogsQL duration forms (1h30m, 1d, 2w) and bare +// seconds; Go's parser knows neither d nor w. +func parseDuration(s string) (time.Duration, error) { + // Bare seconds only as digits with an optional fraction: ParseFloat + // would also take signs, exponents, Inf and NaN. + if whole, frac, ok := strings.Cut(s, "."); isDigits(whole) && (!ok || isDigits(frac)) { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("invalid duration %q", s) + } + d, ok := scaleDuration(f, time.Second) + if !ok { + return 0, fmt.Errorf("duration %q out of range", s) + } + return d, nil + } + if s == "" { + return 0, fmt.Errorf("empty duration") + } + var total time.Duration + for rest := s; rest != ""; { + m := durationToken.FindStringSubmatch(rest) + if m == nil { + return 0, fmt.Errorf("invalid duration %q", s) + } + f, _ := strconv.ParseFloat(m[1], 64) + d, ok := scaleDuration(f, durationUnits[m[2]]) + if !ok || total > math.MaxInt64-d { + return 0, fmt.Errorf("duration %q out of range", s) + } + total += d + rest = rest[len(m[0]):] + } + return total, nil +} + +// scaleDuration refuses what does not fit a Duration: Go leaves an +// out-of-range float to int conversion implementation-defined. +func scaleDuration(f float64, unit time.Duration) (time.Duration, bool) { + v := f * float64(unit) + // float64(math.MaxInt64) rounds up to 2^63, which is already too large. + if v >= float64(math.MaxInt64) { + return 0, false + } + return time.Duration(v), true +} + +// sendableTime refuses a time vlselect cannot hold: it reads the RFC3339 +// the proxy sends through time.UnixNano, which is undefined outside the +// int64 range (1677..2262) and would move the window anywhere. Only times +// actually sent are checked; an older start is clamped first. +func sendableTime(key string, t time.Time) *HTTPError { + if t.Before(time.Unix(0, math.MinInt64)) || t.After(time.Unix(0, math.MaxInt64)) { + return errorf(http.StatusBadRequest, "invalid %s: %s is outside 1677..2262", key, t.UTC().Format(time.RFC3339Nano)) + } + return nil +} + +// parseTime accepts "now", unix seconds with an optional fraction, RFC3339, +// and a duration meaning that long before now (VictoriaLogs' "start=3h"). +// Anything else is refused rather than guessed, since a misread timestamp +// silently moves the window. +func parseTime(s string, now time.Time) (time.Time, error) { + if s == "now" { + return now, nil + } + if sec, frac, ok := strings.Cut(s, "."); isDigits(sec) && (!ok || isDigits(frac)) { + // vlselect reads the unit from the digit count (four digits are a + // year, 13 are milliseconds); only unambiguous seconds are taken. + if len(sec) < 5 || len(sec) > 10 { + return time.Time{}, fmt.Errorf("numeric time %q is not unix seconds", s) + } + secs, err := strconv.ParseInt(sec, 10, 64) + if err != nil { + return time.Time{}, err + } + var nanos int64 + if ok { + frac = (frac + "000000000")[:9] + nanos, _ = strconv.ParseInt(frac, 10, 64) + } + return time.Unix(secs, nanos), nil + } + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + if d, err := parseDuration(s); err == nil { + return now.Add(-d), nil + } + return time.Time{}, fmt.Errorf("unsupported time %q", s) +} + +// hasOptionsBlock reports whether the options keyword is followed by "(", +// skipping whitespace and "#" comments the way the LogsQL lexer does. A +// quoted "options" is never the keyword, but matching it too only refuses +// a search for that literal phrase. +func hasOptionsBlock(query string) bool { + lower := strings.ToLower(query) + for i := strings.Index(lower, "options"); i >= 0; { + rest := lower[i+len("options"):] + for rest != "" { + r, size := utf8.DecodeRuneInString(rest) + switch { + case unicode.IsSpace(r): + rest = rest[size:] + continue + case r == '#': + if n := strings.IndexByte(rest, '\n'); n >= 0 { + rest = rest[n+1:] + } else { + rest = "" + } + continue + } + break + } + if strings.HasPrefix(rest, "(") { + return true + } + next := strings.Index(lower[i+1:], "options") + if next < 0 { + return false + } + i += next + 1 + } + return false +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/internal/observabilityproxy/logsparams_test.go b/internal/observabilityproxy/logsparams_test.go new file mode 100644 index 000000000..82ed29190 --- /dev/null +++ b/internal/observabilityproxy/logsparams_test.go @@ -0,0 +1,434 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +var testNow = time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + +func testLimits() LogsLimits { + return LogsLimits{ + DefaultRange: time.Hour, + MaxRange: 7 * 24 * time.Hour, + MaxBuckets: 1000, + DefaultLimit: 1000, + MaxLimit: 5000, + MaxTailResume: 15 * time.Minute, + } +} + +const wantScope = `{kubernetes_namespace_name="tenant-a"}` + +func build(t *testing.T, endpoint string, in url.Values) url.Values { + t.Helper() + out, err := BuildLogsParams(endpoint, in, "tenant-a", testNow, testLimits()) + if err != nil { + t.Fatalf("BuildLogsParams(%s, %v) error: %v", endpoint, in, err) + } + return out +} + +func buildErr(t *testing.T, endpoint string, in url.Values) *HTTPError { + t.Helper() + out, err := BuildLogsParams(endpoint, in, "tenant-a", testNow, testLimits()) + if err == nil { + t.Fatalf("BuildLogsParams(%s, %v) = %v, want error", endpoint, in, out) + } + return err +} + +func TestBuildLogsParamsAllowlist(t *testing.T) { + allowed := map[string][]string{ + "query": {"query", "start", "end", "limit"}, + "hits": {"query", "start", "end", "step", "field", "fields_limit"}, + "stats_query_range": {"query", "start", "end", "step"}, + "field_values": {"query", "start", "end", "field", "limit", "filter"}, + "stream_field_values": {"query", "start", "end", "field", "limit", "filter"}, + "facets": {"query", "start", "end", "limit", "max_values_per_field", "max_value_len", "keep_const_fields"}, + "tail": {"query", "start", "end", "start_offset"}, + } + hostile := url.Values{ + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_filters[]": {`kubernetes_namespace_name:~".*"`}, + "extra_stream_filters": {`{kubernetes_namespace_name=~".*"}`}, + "AccountID": {"7"}, + "ProjectID": {"7"}, + "tenant": {"tenant-b"}, + "offset": {"1h"}, + "refresh_interval": {"1ms"}, + "ignore_pipes": {"1"}, + "format": {"csv"}, + "timeout": {"1h"}, + "query": {"error"}, + "field": {"kubernetes_pod_name"}, + "filter": {"api"}, + "keep_const_fields": {"1"}, + "since": {"2026-11-04T11:59:00Z"}, + } + for endpoint, keys := range allowed { + t.Run(endpoint, func(t *testing.T) { + out := build(t, endpoint, hostile) + want := map[string]bool{"extra_stream_filters": true} + for _, k := range keys { + want[k] = true + } + for k := range out { + if !want[k] { + t.Errorf("forwarded non-allowlisted parameter %q", k) + } + } + if got := out["extra_stream_filters"]; len(got) != 1 || got[0] != wantScope { + t.Errorf("extra_stream_filters = %q, want exactly [%s]", got, wantScope) + } + if _, ok := out["extra_filters"]; ok { + t.Error("extra_filters forwarded") + } + }) + } +} + +func TestBuildLogsParamsRejects(t *testing.T) { + tests := []struct { + name string + endpoint string + in url.Values + code int + }{ + {"unknown endpoint", "tenant_ids", url.Values{}, http.StatusNotFound}, + // Malformed rather than a request for the default. + {"empty fields_limit", "hits", url.Values{"field": {"level"}, "fields_limit": {""}}, http.StatusBadRequest}, + {"empty limit", "query", url.Values{"limit": {""}}, http.StatusBadRequest}, + {"empty facets cap", "facets", url.Values{"max_values_per_field": {""}}, http.StatusBadRequest}, + {"stats_query is not proxied", "stats_query", url.Values{}, http.StatusNotFound}, + {"ignore_global_time_filter", "query", url.Values{"query": {`* options(ignore_global_time_filter=true)`}}, http.StatusBadRequest}, + {"ignore_global_time_filter in subquery", "hits", url.Values{"query": {`* | union (options(ignore_global_time_filter=true) error)`}}, http.StatusBadRequest}, + {"ignore_global_time_filter on tail", "tail", url.Values{"query": {`options(IGNORE_GLOBAL_TIME_FILTER=true) *`}}, http.StatusBadRequest}, + {"options with a quoted escaped name", "query", url.Values{"query": {`options("ignore_global_time_filte\x72"=true) *`}}, http.StatusBadRequest}, + {"options with a comment inside the name", "query", url.Values{"query": {"options(ignore_global#c\n_time_filter=true) *"}}, http.StatusBadRequest}, + {"options after a comment and unicode space", "tail", url.Values{"query": {"OPTIONS #c\n\u00a0(concurrency=1000) *"}}, http.StatusBadRequest}, + {"inverted range", "query", url.Values{"start": {"2026-11-04T11:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, http.StatusBadRequest}, + {"inverted range on tail", "tail", url.Values{"start": {"2026-11-04T11:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, http.StatusBadRequest}, + {"bad start", "query", url.Values{"start": {"yesterday"}}, http.StatusBadRequest}, + {"millisecond timestamp", "query", url.Values{"start": {"1793786400000"}}, http.StatusBadRequest}, + {"year-like number", "query", url.Values{"start": {"2026"}}, http.StatusBadRequest}, + {"exponent time", "query", url.Values{"start": {"1.7937864e9"}}, http.StatusBadRequest}, + {"signed time", "query", url.Values{"start": {"+1793786400"}}, http.StatusBadRequest}, + {"exponent duration", "query", url.Values{"start": {"3e2"}}, http.StatusBadRequest}, + {"trailing dot", "query", url.Values{"start": {"300."}}, http.StatusBadRequest}, + {"infinite time", "query", url.Values{"end": {"Inf"}}, http.StatusBadRequest}, + // vlselect keeps times as int64 nanoseconds (time.UnixNano), which + // is undefined outside 1677..2262; a time sent there must fit. + {"end before int64 nanoseconds", "query", url.Values{"start": {"0001-01-01T00:00:00Z"}, "end": {"0001-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"end past int64 nanoseconds", "query", url.Values{"end": {"9999-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"unix seconds past int64 nanoseconds", "query", url.Values{"start": {"9999999990"}, "end": {"9999999999"}}, http.StatusBadRequest}, + {"tail end past int64 nanoseconds", "tail", url.Values{"end": {"9999-01-02T00:00:00Z"}}, http.StatusBadRequest}, + {"NaN time", "query", url.Values{"start": {"NaN"}}, http.StatusBadRequest}, + {"NaN step", "hits", url.Values{"step": {"NaN"}}, http.StatusBadRequest}, + {"NaN start_offset", "tail", url.Values{"start_offset": {"NaN"}}, http.StatusBadRequest}, + {"bad limit", "query", url.Values{"limit": {"-1"}}, http.StatusBadRequest}, + {"zero limit", "query", url.Values{"limit": {"0"}}, http.StatusBadRequest}, + {"bad step", "hits", url.Values{"step": {"0s"}}, http.StatusBadRequest}, + {"field required", "field_values", url.Values{}, http.StatusBadRequest}, + {"too many hits fields", "hits", url.Values{"field": strings.Fields("a b c d e f g h i j k")}, http.StatusBadRequest}, + {"since and start_offset", "tail", url.Values{"since": {"2026-11-04T11:59:00Z"}, "start_offset": {"1m"}}, http.StatusBadRequest}, + {"bad since", "tail", url.Values{"since": {"soon"}}, http.StatusBadRequest}, + {"start_offset summing past int64", "tail", url.Values{"start_offset": {"99999999999h99999999999h1s"}}, http.StatusBadRequest}, + {"step summing past int64", "hits", url.Values{"step": {"2562047h2562047h"}}, http.StatusBadRequest}, + {"step token past int64", "hits", url.Values{"step": {"99999999999h"}}, http.StatusBadRequest}, + {"bare-seconds step past int64", "hits", url.Values{"step": {"99999999999999999999"}}, http.StatusBadRequest}, + {"relative start summing past int64", "query", url.Values{"start": {"106751d1d"}}, http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := buildErr(t, tt.endpoint, tt.in); err.Code != tt.code { + t.Fatalf("code = %d (%s), want %d", err.Code, err.Message, tt.code) + } + }) + } +} + +// VictoriaLogs propagates extra_stream_filters into every subquery +// (docs.victoriametrics.com/victorialogs/querying/), so union and in() +// queries are forwarded verbatim with the scope beside them. +func TestBuildLogsParamsSubqueriesKeepScope(t *testing.T) { + for _, q := range []string{ + `* | union ({kubernetes_namespace_name="tenant-b"})`, + `_stream_id:in({kubernetes_namespace_name="tenant-b"} | fields _stream_id)`, + } { + out := build(t, "query", url.Values{"query": {q}}) + if out.Get("query") != q || out.Get("extra_stream_filters") != wantScope { + t.Errorf("query %q forwarded as %v", q, out) + } + } +} + +func TestBuildLogsParamsWindow(t *testing.T) { + ts := func(s string) string { + v, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatal(err) + } + return v.UTC().Format(time.RFC3339Nano) + } + tests := []struct { + name string + in url.Values + wantStart, wantEnd string + }{ + {"defaults", url.Values{}, ts("2026-11-04T11:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"start only", url.Values{"start": {"2026-11-04T09:00:00Z"}}, ts("2026-11-04T09:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"end only", url.Values{"end": {"2026-11-03T00:00:00Z"}}, ts("2026-11-02T23:00:00Z"), ts("2026-11-03T00:00:00Z")}, + {"unix seconds", url.Values{"start": {"1793786400"}, "end": {"1793788200.5"}}, "2026-11-04T10:00:00Z", "2026-11-04T10:30:00.5Z"}, + {"rfc3339 with offset", url.Values{"start": {"2026-11-04T13:00:00+03:00"}}, ts("2026-11-04T10:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"relative", url.Values{"start": {"3h"}, "end": {"now"}}, ts("2026-11-04T09:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"relative days", url.Values{"start": {"1d"}}, ts("2026-11-03T12:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"over-wide clamped", url.Values{"start": {"2026-01-01T00:00:00Z"}}, ts("2026-10-28T12:00:00Z"), ts("2026-11-04T12:00:00Z")}, + {"empty range allowed", url.Values{"start": {"2026-11-04T10:00:00Z"}, "end": {"2026-11-04T10:00:00Z"}}, ts("2026-11-04T10:00:00Z"), ts("2026-11-04T10:00:00Z")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, endpoint := range []string{"query", "field_values", "facets"} { + in := url.Values{"field": {"f"}} + for k, v := range tt.in { + in[k] = v + } + out := build(t, endpoint, in) + if out.Get("start") != tt.wantStart || out.Get("end") != tt.wantEnd { + t.Errorf("%s: window = [%s, %s], want [%s, %s]", endpoint, out.Get("start"), out.Get("end"), tt.wantStart, tt.wantEnd) + } + } + }) + } +} + +func TestBuildLogsParamsStepAndLimit(t *testing.T) { + day := url.Values{"start": {"2026-11-03T12:00:00Z"}} + with := func(base url.Values, k, v string) url.Values { + c := url.Values{} + for key, vals := range base { + c[key] = vals + } + c.Set(k, v) + return c + } + // 24h over 999 steps, 1000 buckets once aligned, is 86486.5ms, rounded + // up to whole milliseconds. + if got := build(t, "hits", with(day, "step", "1s")).Get("step"); got != "86487ms" { + t.Errorf("tiny step = %s, want clamped to 86487ms", got) + } + if got := build(t, "stats_query_range", with(day, "step", "1h")).Get("step"); got != "3600000ms" { + t.Errorf("1h step = %s", got) + } + if got := build(t, "stats_query_range", with(day, "step", "1w")).Get("step"); got != "86400000ms" { + t.Errorf("1w step on a day window = %s, want capped at the window", got) + } + if got := build(t, "hits", with(day, "step", "3600")).Get("step"); got != "3600000ms" { + t.Errorf("bare-seconds step = %s, want 3600000ms", got) + } + if got := build(t, "hits", with(day, "step", "5400.5")).Get("step"); got != "5400500ms" { + t.Errorf("fractional bare-seconds step = %s, want 5400500ms", got) + } + if got := build(t, "hits", day).Get("step"); got != "86487ms" { + t.Errorf("omitted step = %s, want the bucket-bounded minimum", got) + } + if got := build(t, "hits", url.Values{"start": {"2026-11-04T11:59:59Z"}}).Get("step"); got != "1000ms" { + t.Errorf("omitted step on a tiny window = %s, want 1000ms floor", got) + } + + if got := build(t, "query", url.Values{}).Get("limit"); got != "1000" { + t.Errorf("default limit = %s", got) + } + if got := build(t, "query", url.Values{"limit": {"10"}}).Get("limit"); got != "10" { + t.Errorf("limit = %s", got) + } + for _, endpoint := range []string{"query", "field_values", "facets"} { + in := url.Values{"limit": {"1000000"}, "field": {"f"}} + if got := build(t, endpoint, in).Get("limit"); got != "5000" { + t.Errorf("%s: limit = %s, want clamped to 5000", endpoint, got) + } + } + facets := build(t, "facets", url.Values{"max_values_per_field": {"99999"}, "max_value_len": {"99999"}}) + if facets.Get("max_values_per_field") != "5000" || facets.Get("max_value_len") != "5000" { + t.Errorf("facets caps not clamped: %v", facets) + } + // Without fields_limit vlselect returns every group, so the cap is + // sent even when the client sends none. + if got := build(t, "hits", url.Values{"field": {"level"}}).Get("fields_limit"); got != "5000" { + t.Errorf("hits without fields_limit forwards %q, want the 5000 cap", got) + } + hits := build(t, "hits", url.Values{"field": {"b", "a"}, "fields_limit": {"99999"}}) + fields := hits["field"] + sort.Strings(fields) + if strings.Join(fields, ",") != "a,b" || hits.Get("fields_limit") != "5000" { + t.Errorf("hits fields = %v", hits) + } +} + +func TestBuildLogsParamsTail(t *testing.T) { + tests := []struct { + name string + in url.Values + want string + }{ + {"live only", url.Values{}, ""}, + // now - 5s tail offset - since = 55s + {"since within cap", url.Values{"since": {"2026-11-04T11:59:00Z"}}, "55000ms"}, + {"since older than cap", url.Values{"since": {"2026-11-04T08:00:00Z"}}, "900000ms"}, + {"since in the future", url.Values{"since": {"2026-11-04T13:00:00Z"}}, "0ms"}, + {"start_offset within cap", url.Values{"start_offset": {"2m"}}, "120000ms"}, + {"start_offset over cap", url.Values{"start_offset": {"30d"}}, "900000ms"}, + {"start_offset in bare seconds", url.Values{"start_offset": {"120"}}, "120000ms"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := build(t, "tail", tt.in) + if got := out.Get("start_offset"); got != tt.want { + t.Errorf("start_offset = %q, want %q", got, tt.want) + } + if out.Has("start") || out.Has("end") || out.Has("limit") { + t.Errorf("tail got a derived window or limit: %v", out) + } + }) + } + out := build(t, "tail", url.Values{"start": {"2026-11-04T11:00:00Z"}}) + if out.Get("start") != "2026-11-04T11:00:00Z" { + t.Errorf("tail start = %q, want the client value ANDed into each tick", out.Get("start")) + } +} + +// alignedWindow repeats vlselect's alignStartEndToStep (offset 0): with a +// step it widens [start, end] to step boundaries, end always by at least +// one more step. +func alignedWindow(t *testing.T, out url.Values) time.Duration { + t.Helper() + parse := func(k string) int64 { + v, err := time.Parse(time.RFC3339Nano, out.Get(k)) + if err != nil { + t.Fatalf("%s: %v", k, err) + } + return v.UnixNano() + } + step, err := parseDuration(out.Get("step")) + if err != nil { + t.Fatal(err) + } + start, end, st := parse("start"), parse("end"), int64(step) + start -= start % st + end += st - end%st + return time.Duration(end - start) +} + +func TestBuildLogsParamsStepCannotWidenWindow(t *testing.T) { + lim := testLimits() + for _, endpoint := range []string{"hits", "stats_query_range"} { + for _, in := range []url.Values{ + {"step": {"100y"}}, + {"step": {"100y"}, "start": {"2026-01-01T00:00:00Z"}}, + {"step": {"3d"}, "start": {"7d"}}, + {"start": {"7d"}}, + {"step": {"1s"}, "start": {"7d"}}, + } { + out := build(t, endpoint, in) + if got := alignedWindow(t, out); got > lim.MaxRange { + t.Errorf("%s %v: vlselect would scan %s, more than %s", endpoint, in, got, lim.MaxRange) + } + } + } +} + +// TestBuildLogsParamsStepKeepsBucketCap counts buckets the way vlselect +// returns them: one per step boundary from the aligned start through the +// aligned end, zero-filled, which is one more than the unaligned span +// holds whenever start is not on a boundary. +func TestBuildLogsParamsStepKeepsBucketCap(t *testing.T) { + lim := testLimits() + // Spans that MaxBuckets does not divide into whole milliseconds, and + // unaligned starts of spans it does divide. + for _, start := range []string{ + "2026-11-03T12:00:00.0007Z", "2026-11-04T10:00:00.0013Z", "2026-11-04T11:00:00.0005Z", + "2026-11-01T12:00:00.5Z", "2026-11-04T11:00:00.25Z", + } { + out := build(t, "hits", url.Values{"start": {start}}) + step, err := parseDuration(out.Get("step")) + if err != nil { + t.Fatal(err) + } + if buckets := alignedWindow(t, out) / step; int(buckets) > lim.MaxBuckets { + t.Errorf("start %s: step %s gives %d buckets, more than %d", start, out.Get("step"), buckets, lim.MaxBuckets) + } + } +} + +func TestParseDurationBounds(t *testing.T) { + for _, in := range []string{"2562047h", "106751d", "9223372036s", "292y"} { + if d, err := parseDuration(in); err != nil || d <= 0 { + t.Errorf("parseDuration(%q) = %s, %v; want a positive duration", in, d, err) + } + } + for _, in := range []string{"2562048h", "106751d1d", "9223372037s", "9223372037", "293y", "1y292y"} { + if d, err := parseDuration(in); err == nil { + t.Errorf("parseDuration(%q) = %s; want an error past the int64 range", in, d) + } + } +} + +// TestBuildLogsParamsOldTimesClamped: only times that reach vlselect must +// fit its int64 nanoseconds. An old start is clamped to MaxRange before it +// is sent, and an old since becomes a start_offset capped at the resume +// limit, so both are served. +func TestBuildLogsParamsOldTimesClamped(t *testing.T) { + lim := testLimits() + out := build(t, "query", url.Values{"start": {"1900-01-01T00:00:00Z"}}) + if want := testNow.Add(-lim.MaxRange).Format(time.RFC3339Nano); out.Get("start") != want { + t.Errorf("start = %s, want clamped to %s", out.Get("start"), want) + } + out = build(t, "tail", url.Values{"since": {"1969-12-31T23:59:59Z"}}) + if got := out.Get("start_offset"); got != formatDuration(lim.MaxTailResume) { + t.Errorf("start_offset = %s, want the %s cap", got, lim.MaxTailResume) + } +} + +// TestBuildLogsParamsSendableBoundary pins the inclusive int64-nanosecond +// bound and that the refusal names the instant exactly. +func TestBuildLogsParamsSendableBoundary(t *testing.T) { + last := time.Unix(0, math.MaxInt64).UTC() + in := url.Values{"start": {last.Add(-time.Hour).Format(time.RFC3339Nano)}, "end": {last.Format(time.RFC3339Nano)}} + if _, herr := BuildLogsParams("query", in, "tenant-a", testNow, testLimits()); herr != nil { + t.Fatalf("the last int64 nanosecond refused: %v", herr) + } + past := last.Add(time.Nanosecond).Format(time.RFC3339Nano) + in = url.Values{"start": {last.Add(-time.Hour).Format(time.RFC3339Nano)}, "end": {past}} + _, herr := BuildLogsParams("query", in, "tenant-a", testNow, testLimits()) + if herr == nil || !strings.Contains(herr.Message, "invalid end") || !strings.Contains(herr.Message, past) { + t.Fatalf("one nanosecond past answered %v, want end refused naming %s", herr, past) + } + // Both out of range: the answer names end, deterministically. + for range 20 { + _, herr := BuildLogsParams("query", url.Values{"start": {"0001-01-01T00:00:00Z"}, "end": {"0001-01-02T00:00:00Z"}}, "tenant-a", testNow, testLimits()) + if herr == nil || !strings.Contains(herr.Message, "invalid end") { + t.Fatalf("answer = %v, want end named", herr) + } + } +} diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go new file mode 100644 index 000000000..56ee9e496 --- /dev/null +++ b/internal/observabilityproxy/route.go @@ -0,0 +1,148 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "net/http" + "strings" + + authorizationv1 "k8s.io/api/authorization/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +// RouteKind is one of the four Kubernetes-shaped roots the proxy serves. +type RouteKind int + +const ( + NamespaceLogs RouteKind = iota + 1 + NamespaceMetrics + ApplicationLog + ApplicationMetrics +) + +func (k RouteKind) String() string { + switch k { + case NamespaceLogs: + return "namespace-logs" + case NamespaceMetrics: + return "namespace-metrics" + case ApplicationLog: + return "application-log" + case ApplicationMetrics: + return "application-metrics" + } + return "unknown" +} + +const ( + coreGroup = "cozystack.io" + appsGroup = "apps.cozystack.io" +) + +// Route is a request path split into the scope it names and the upstream +// endpoint below the root. Plural and Name are set only on application roots. +type Route struct { + Kind RouteKind + Namespace string + Plural string + Name string + Endpoint string +} + +// HTTPError carries the status code the proxy answers with. +type HTTPError struct { + Code int + Message string +} + +func (e *HTTPError) Error() string { return fmt.Sprintf("%d: %s", e.Code, e.Message) } + +func errorf(code int, format string, args ...any) *HTTPError { + return &HTTPError{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// ParseRoute takes the escaped request path. The console nginx matches its +// location on the normalized path but forwards the original one, so a path +// that a normalizer would rewrite (dot segments, empty segments, any +// percent-encoding such as %2F or %2e) is refused instead of interpreted: +// the namespace the SubjectAccessReview names must be the one queried. +func ParseRoute(escapedPath string) (Route, *HTTPError) { + notFound := errorf(http.StatusNotFound, "not found") + if strings.Contains(escapedPath, "%") { + return Route{}, errorf(http.StatusBadRequest, "percent-encoded paths are not accepted") + } + segs := strings.Split(strings.TrimPrefix(escapedPath, "/"), "/") + if len(segs) > 0 && segs[len(segs)-1] == "" { + segs = segs[:len(segs)-1] + } + for _, s := range segs { + if s == "" || s == "." || s == ".." { + return Route{}, errorf(http.StatusBadRequest, "non-canonical path") + } + } + if len(segs) < 6 || segs[0] != "apis" || segs[2] != "v1alpha1" || segs[3] != "namespaces" { + return Route{}, notFound + } + + var r Route + var rest []string + switch { + case segs[1] == coreGroup && segs[5] == "logs": + r.Kind, rest = NamespaceLogs, segs[6:] + case segs[1] == coreGroup && segs[5] == "metrics": + r.Kind, rest = NamespaceMetrics, segs[6:] + case segs[1] == appsGroup && len(segs) >= 8 && segs[7] == "log": + r.Kind, rest = ApplicationLog, segs[8:] + case segs[1] == appsGroup && len(segs) >= 8 && segs[7] == "metrics": + r.Kind, rest = ApplicationMetrics, segs[8:] + default: + return Route{}, notFound + } + + r.Namespace = segs[4] + if errs := validation.IsDNS1123Label(r.Namespace); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid namespace %q: %s", r.Namespace, strings.Join(errs, "; ")) + } + if r.Kind == ApplicationLog || r.Kind == ApplicationMetrics { + r.Plural, r.Name = segs[5], segs[6] + if errs := validation.IsDNS1123Label(r.Plural); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid resource %q: %s", r.Plural, strings.Join(errs, "; ")) + } + if errs := validation.IsDNS1123Subdomain(r.Name); len(errs) > 0 { + return Route{}, errorf(http.StatusBadRequest, "invalid name %q: %s", r.Name, strings.Join(errs, "; ")) + } + } + r.Endpoint = strings.Join(rest, "/") + return r, nil +} + +// ResourceAttributes is the single SubjectAccessReview a route requires. It +// depends on the path only, so a POST query authorizes exactly like a GET. +func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { + switch r.Kind { + case NamespaceLogs: + return authorizationv1.ResourceAttributes{Verb: "list", Group: coreGroup, Resource: "logs", Namespace: r.Namespace} + case NamespaceMetrics: + return authorizationv1.ResourceAttributes{Verb: "list", Group: coreGroup, Resource: "metrics", Namespace: r.Namespace} + case ApplicationLog: + return authorizationv1.ResourceAttributes{Verb: "get", Group: appsGroup, Resource: r.Plural, Subresource: "log", Name: r.Name, Namespace: r.Namespace} + case ApplicationMetrics: + return authorizationv1.ResourceAttributes{Verb: "get", Group: appsGroup, Resource: r.Plural, Subresource: "metrics", Name: r.Name, Namespace: r.Namespace} + } + return authorizationv1.ResourceAttributes{} +} diff --git a/internal/observabilityproxy/route_test.go b/internal/observabilityproxy/route_test.go new file mode 100644 index 000000000..f34311e0f --- /dev/null +++ b/internal/observabilityproxy/route_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "testing" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +func TestParseRoute(t *testing.T) { + tests := []struct { + name string + path string + wantCode int + want Route + }{ + { + name: "namespace logs", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/logsql/query", + want: Route{Kind: NamespaceLogs, Namespace: "tenant-a", Endpoint: "select/logsql/query"}, + }, + { + name: "namespace logs root without endpoint", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", + want: Route{Kind: NamespaceLogs, Namespace: "tenant-a"}, + }, + { + name: "namespace metrics", + path: "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", + want: Route{Kind: NamespaceMetrics, Namespace: "tenant-a", Endpoint: "api/v1/query"}, + }, + { + name: "application log", + path: "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db.main/log/select/logsql/tail", + want: Route{Kind: ApplicationLog, Namespace: "tenant-a", Plural: "postgreses", Name: "db.main", Endpoint: "select/logsql/tail"}, + }, + { + name: "application metrics", + path: "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range", + want: Route{Kind: ApplicationMetrics, Namespace: "tenant-a", Plural: "postgreses", Name: "db", Endpoint: "api/v1/query_range"}, + }, + {name: "unknown group", path: "/apis/core.cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusNotFound}, + {name: "unknown version", path: "/apis/cozystack.io/v1/namespaces/a/logs/x", wantCode: http.StatusNotFound}, + {name: "unknown namespace resource", path: "/apis/cozystack.io/v1alpha1/namespaces/a/events/x", wantCode: http.StatusNotFound}, + {name: "unknown app subresource", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/db/exec/x", wantCode: http.StatusNotFound}, + {name: "core api", path: "/api/v1/namespaces/a/pods", wantCode: http.StatusNotFound}, + {name: "truncated", path: "/apis/cozystack.io/v1alpha1/namespaces/a", wantCode: http.StatusNotFound}, + {name: "empty segment", path: "/apis/cozystack.io/v1alpha1/namespaces/a/logs/select//logsql/query", wantCode: http.StatusBadRequest}, + {name: "leading double slash", path: "//apis/cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot dot segment", path: "/apis/cozystack.io/v1alpha1/namespaces/b/../a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot dot under root", path: "/apis/cozystack.io/v1alpha1/namespaces/a/logs/../metrics/x", wantCode: http.StatusBadRequest}, + {name: "dot dot in unknown prefix", path: "/foo/../apis/cozystack.io/v1alpha1/namespaces/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "dot segment", path: "/apis/cozystack.io/v1alpha1/namespaces/a/./logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded slash", path: "/apis/cozystack.io/v1alpha1/namespaces/a%2Fb/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded slash lowercase", path: "/apis/cozystack.io/v1alpha1/namespaces/a%2fb/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded dot dot", path: "/apis/cozystack.io/v1alpha1/namespaces/b/%2e%2e/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "encoded dot dot uppercase", path: "/apis/cozystack.io/v1alpha1/namespaces/b/%2E%2E/a/logs/x", wantCode: http.StatusBadRequest}, + {name: "backslash", path: `/apis/cozystack.io/v1alpha1/namespaces/b\..\a/logs/x`, wantCode: http.StatusBadRequest}, + {name: "uppercase namespace", path: "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/x", wantCode: http.StatusBadRequest}, + {name: "namespace with dot", path: "/apis/cozystack.io/v1alpha1/namespaces/a.b/logs/x", wantCode: http.StatusBadRequest}, + {name: "namespace with quote", path: `/apis/cozystack.io/v1alpha1/namespaces/a"b/logs/x`, wantCode: http.StatusBadRequest}, + {name: "name with logsql metacharacters", path: `/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/db"|union/log/x`, wantCode: http.StatusBadRequest}, + {name: "name with braces", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgreses/{db}/log/x", wantCode: http.StatusBadRequest}, + {name: "plural uppercase", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/Postgreses/db/log/x", wantCode: http.StatusBadRequest}, + {name: "plural with dot", path: "/apis/apps.cozystack.io/v1alpha1/namespaces/a/postgres.es/db/log/x", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRoute(tt.path) + if tt.wantCode != 0 { + if err == nil { + t.Fatalf("ParseRoute(%q) = %+v, want error with code %d", tt.path, got, tt.wantCode) + } + if err.Code != tt.wantCode { + t.Fatalf("ParseRoute(%q) code = %d (%s), want %d", tt.path, err.Code, err.Message, tt.wantCode) + } + return + } + if err != nil { + t.Fatalf("ParseRoute(%q) error = %v", tt.path, err) + } + if got != tt.want { + t.Fatalf("ParseRoute(%q) = %+v, want %+v", tt.path, got, tt.want) + } + }) + } +} + +func TestRouteResourceAttributes(t *testing.T) { + tests := []struct { + route Route + want authorizationv1.ResourceAttributes + }{ + { + route: Route{Kind: NamespaceLogs, Namespace: "a"}, + want: authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "logs", Namespace: "a"}, + }, + { + route: Route{Kind: NamespaceMetrics, Namespace: "a"}, + want: authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "a"}, + }, + { + route: Route{Kind: ApplicationLog, Namespace: "a", Plural: "postgreses", Name: "db"}, + want: authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db", Namespace: "a"}, + }, + { + route: Route{Kind: ApplicationMetrics, Namespace: "a", Plural: "postgreses", Name: "db"}, + want: authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db", Namespace: "a"}, + }, + } + for _, tt := range tests { + if got := tt.route.ResourceAttributes(); got != tt.want { + t.Errorf("%v.ResourceAttributes() = %+v, want %+v", tt.route.Kind, got, tt.want) + } + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go new file mode 100644 index 000000000..e980818d9 --- /dev/null +++ b/internal/observabilityproxy/server.go @@ -0,0 +1,388 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// tailChunkBytes bounds the memory one tail holds: a longer line is passed +// on in pieces and flushed once its end arrives. +const tailChunkBytes = 64 << 10 + +// errTailIdle cancels a tail that received nothing for the idle timeout; +// it wraps DeadlineExceeded so a store that never answered reads as 504. +var errTailIdle = fmt.Errorf("tail idle: %w", context.DeadlineExceeded) + +// errAborted ends a response whose status line already went out by +// cutting the connection, so a client sees a broken body instead of a +// complete one with lines missing. +var errAborted = &HTTPError{Code: http.StatusBadGateway, Message: "response aborted"} + +// maxFormBytes bounds a POST body; LogsQL queries are a few KiB at most. +const maxFormBytes = 1 << 20 + +// Config is the runtime configuration of the proxy handler. +type Config struct { + ClusterDomain string + // UpstreamURL maps a store namespace to the vlselect base URL; nil + // means the in-cluster Service of the "generic" logs storage. + UpstreamURL func(store string) string + Logs LogsLimits + Limiter LimiterConfig + RequestTimeout time.Duration + TailIdleTimeout time.Duration +} + +// StoreLookup resolves a request namespace to the namespace of its store. +type StoreLookup interface { + Resolve(namespace string) (string, error) +} + +// Server serves the tenant observability roots. +type Server struct { + cfg Config + auth *Authorizer + stores StoreLookup + client *http.Client + limits *limiter + now func() time.Time + requests *prometheus.CounterVec +} + +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { + if cfg.UpstreamURL == nil { + domain := cfg.ClusterDomain + cfg.UpstreamURL = func(store string) string { + return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain) + } + } + s := &Server{ + cfg: cfg, + auth: auth, + stores: stores, + // No client timeout: it would cut live tails. Non-tail requests are + // bounded by a context deadline instead. + client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, + limits: newLimiter(cfg.Limiter), + now: time.Now, + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "observability_proxy_requests_total", + Help: "Requests served, by route kind and status code.", + }, []string{"route", "code"}), + } + if reg != nil { + reg.MustRegister(s.requests, prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "observability_proxy_open_tails", + Help: "Live tails currently open.", + }, func() float64 { return float64(s.limits.openTails()) })) + } + return s +} + +type statusRecorder struct { + http.ResponseWriter + code int + // bodyRead is set once readForm has read the whole body. + bodyRead bool +} + +func (r *statusRecorder) WriteHeader(code int) { + if r.code == 0 { + r.code = code + } + r.ResponseWriter.WriteHeader(code) +} + +func (r *statusRecorder) Write(b []byte) (int, error) { + if r.code == 0 { + r.code = http.StatusOK + } + return r.ResponseWriter.Write(b) +} + +func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter } + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rec := &statusRecorder{ResponseWriter: w} + // The listener has no ReadTimeout, which would cut tails, so the body + // is bounded from here; readForm lifts the deadline once it has read + // the body. A refusal sent before that leaves the deadline in place. + if s.cfg.RequestTimeout > 0 { + _ = http.NewResponseController(w).SetReadDeadline(time.Now().Add(s.cfg.RequestTimeout)) + } + route, herr := ParseRoute(r.URL.EscapedPath()) + if herr == nil { + herr = s.serve(rec, r, route) + } + if herr != nil && herr != errAborted { + // net/http drains up to 256 KiB of an unread body before it sends + // the headers; closing the connection instead answers a refusal at + // once rather than after a body that may never come. + if r.ContentLength != 0 && !rec.bodyRead { + rec.Header().Set("Connection", "close") + } + writeError(rec, herr) + } + s.requests.WithLabelValues(route.Kind.String(), strconv.Itoa(rec.code)).Inc() + if herr == errAborted { + panic(http.ErrAbortHandler) + } +} + +func writeError(w http.ResponseWriter, e *HTTPError) { + if e.Code == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Bearer realm="cozystack"`) + } + http.Error(w, e.Message, e.Code) +} + +// serve answers a root it has no handler for before authenticating, so an +// unknown path costs no review calls. +func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { + if route.Kind != NamespaceLogs { + return errorf(http.StatusNotFound, "not found") + } + user, herr := s.authorize(r, route) + if herr != nil { + return herr + } + if r.Method != http.MethodGet && r.Method != http.MethodPost { + w.Header().Set("Allow", "GET, POST") + return errorf(http.StatusMethodNotAllowed, "method not allowed") + } + if wait, ok := s.limits.allowRate(user); !ok { + w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) + return errorf(http.StatusTooManyRequests, "request rate limit exceeded") + } + endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") + if !ok { + return errorf(http.StatusNotFound, "not found") + } + tail := endpoint == "tail" + acquire := s.limits.acquire + if tail { + acquire = s.limits.acquireTail + } + release, ok := acquire(user) + if !ok { + return tooManyConcurrent(w) + } + defer release() + if herr := s.readForm(w, r); herr != nil { + return herr + } + params, herr := BuildLogsParams(endpoint, r.Form, route.Namespace, s.now(), s.cfg.Logs) + if herr != nil { + return herr + } + + store, err := s.stores.Resolve(route.Namespace) + if err != nil { + slog.Error("resolving log store", "namespace", route.Namespace, "error", err) + return errorf(http.StatusServiceUnavailable, "cannot resolve the log store") + } + + ctx, cancel := context.WithCancelCause(r.Context()) + defer cancel(nil) + limit := s.cfg.RequestTimeout + var idle *time.Timer + if tail { + // Started before the request, so a store that never sends the + // headers is bounded as well as one that stops sending lines. + if limit = s.cfg.TailIdleTimeout; limit > 0 { + idle = time.AfterFunc(limit, func() { cancel(errTailIdle) }) + defer idle.Stop() + } + } else if limit > 0 { + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, limit) + defer stop() + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode())) + if err != nil { + return errorf(http.StatusInternalServerError, "building upstream request: %v", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("AccountID", "0") + req.Header.Set("ProjectID", "0") + if tail { + // vlselect holds even the headers of a gzip-encoded tail until the + // first line; an explicit value stops the transport from asking for gzip. + req.Header.Set("Accept-Encoding", "identity") + } + resp, err := s.client.Do(req) + switch { + case err == nil: + case r.Context().Err() != nil: + return errorf(http.StatusBadGateway, "client went away") + case errors.Is(context.Cause(ctx), context.DeadlineExceeded): + return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit) + default: + slog.Warn("log store unreachable", "store", store, "error", err) + return errorf(http.StatusBadGateway, "log store unreachable") + } + defer func() { _ = resp.Body.Close() }() + + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + // Cancelling the upstream request does not interrupt a write to a + // client that stopped reading; only a write deadline does. A tail + // renews it before every line, so only a stalled client is cut; the + // server clears it once the response is finished. + rc := http.NewResponseController(w) + arm := func() {} + if tail && s.cfg.TailIdleTimeout > 0 { + arm = func() { _ = rc.SetWriteDeadline(time.Now().Add(s.cfg.TailIdleTimeout)) } + } else if deadline, ok := ctx.Deadline(); ok { + _ = rc.SetWriteDeadline(deadline) + } + arm() + w.WriteHeader(resp.StatusCode) + if rc.Flush() != nil { + return nil + } + if tail { + s.streamLines(w, rc, resp.Body, idle, arm) + } else if _, err := io.Copy(w, resp.Body); err != nil { + slog.Debug("copying upstream response", "error", err) + return errAborted + } + return nil +} + +// concurrencyRetryAfter is a hint only: a slot frees when another request +// of the user ends, which the proxy cannot predict, and one second keeps +// a polling client from retrying in a tight loop. +const concurrencyRetryAfter = "1" + +func tooManyConcurrent(w http.ResponseWriter) *HTTPError { + w.Header().Set("Retry-After", concurrencyRetryAfter) + return errorf(http.StatusTooManyRequests, "too many concurrent requests") +} + +// readForm reads the whole body under the request timeout, so a client +// that sends it slowly holds its slot only that long. ParseForm leaves the +// body of a GET or of a POST that is not a form unread, and net/http +// drains a leftover body with no deadline once the headers go out, so the +// rest is read here as well; draining rather than refusing a declared +// body keeps any client that sends one working. Once the body is read the +// deadline is lifted: the server's background read would take a later +// expiry for a client that went away and cancel a tail. After a failed +// read it stays, so the server's drain of the unread body is bounded too. +func (s *Server) readForm(w http.ResponseWriter, r *http.Request) *HTTPError { + r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes) + rc := http.NewResponseController(w) + if s.cfg.RequestTimeout > 0 { + _ = rc.SetReadDeadline(time.Now().Add(s.cfg.RequestTimeout)) + } + err := r.ParseForm() + if err == nil { + _, err = io.Copy(io.Discard, r.Body) + } + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + return errorf(http.StatusRequestEntityTooLarge, "request body larger than %d bytes", tooLarge.Limit) + case err == nil: + _ = rc.SetReadDeadline(time.Time{}) + if rec, ok := w.(*statusRecorder); ok { + rec.bodyRead = true + } + return nil + case errors.Is(err, os.ErrDeadlineExceeded): + return errorf(http.StatusRequestTimeout, "request body not received within %s", s.cfg.RequestTimeout) + default: + return errorf(http.StatusBadRequest, "invalid form: %v", err) + } +} + +func (s *Server) authorize(r *http.Request, route Route) (string, *HTTPError) { + scheme, token, _ := strings.Cut(r.Header.Get("Authorization"), " ") + if !strings.EqualFold(scheme, "Bearer") { + return "", errorf(http.StatusUnauthorized, "missing bearer token") + } + // The kube client sets no timeout of its own, and net/http cancels + // r.Context on a dropped client only once the body is read, so a + // stalled review would otherwise outlive every other limit. + ctx := r.Context() + if s.cfg.RequestTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout) + defer cancel() + } + user, err := s.auth.Authenticate(ctx, strings.TrimSpace(token)) + if errors.Is(err, ErrUnauthenticated) { + return "", errorf(http.StatusUnauthorized, "invalid bearer token") + } + if err != nil { + slog.Error("token review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authentication unavailable") + } + allowed, err := s.auth.Authorize(ctx, user, route.ResourceAttributes()) + if err != nil { + slog.Error("subject access review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") + } + if !allowed { + return "", errorf(http.StatusForbidden, "forbidden") + } + return user.Username, nil +} + +// streamLines flushes after every line so a live tail reaches the browser +// as it happens; each line pushes back idle, which cancels the upstream +// stream once no line has arrived for the idle timeout. +func (s *Server) streamLines(w http.ResponseWriter, rc *http.ResponseController, body io.Reader, idle *time.Timer, arm func()) { + rd := bufio.NewReaderSize(body, tailChunkBytes) + for { + chunk, err := rd.ReadSlice('\n') + if len(chunk) > 0 { + if idle != nil { + idle.Reset(s.cfg.TailIdleTimeout) + } + arm() + if _, werr := w.Write(chunk); werr != nil { + return + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + if ferr := rc.Flush(); ferr != nil { + return + } + } + if err != nil && !errors.Is(err, bufio.ErrBufferFull) { + return + } + } +} diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go new file mode 100644 index 000000000..7415b46fa --- /dev/null +++ b/internal/observabilityproxy/server_deadline_test.go @@ -0,0 +1,446 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// pipeListener hands the server one end of a net.Pipe per dial. A pipe has +// no buffer, so a client that stops reading blocks the server's next write +// at once, as a full TCP window eventually does. +type pipeListener struct { + conns chan net.Conn + closed chan struct{} + once sync.Once +} + +type pipeAddr struct{} + +func (pipeAddr) Network() string { return "pipe" } +func (pipeAddr) String() string { return "pipe" } + +func servePipe(t *testing.T, handler http.Handler) *pipeListener { + t.Helper() + l := &pipeListener{conns: make(chan net.Conn), closed: make(chan struct{})} + srv := &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second} + go func() { _ = srv.Serve(l) }() + t.Cleanup(func() { _ = srv.Close() }) + return l +} + +func (l *pipeListener) Accept() (net.Conn, error) { + select { + case c := <-l.conns: + return c, nil + case <-l.closed: + return nil, net.ErrClosed + } +} + +func (l *pipeListener) Close() error { + l.once.Do(func() { close(l.closed) }) + return nil +} + +func (l *pipeListener) Addr() net.Addr { return pipeAddr{} } + +// send opens a connection and writes raw to it without waiting for the +// server to read all of it. +func (l *pipeListener) send(t *testing.T, raw string) net.Conn { + t.Helper() + client, server := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + select { + case l.conns <- server: + case <-time.After(2 * time.Second): + t.Fatal("pipe server is not accepting") + } + go func() { _, _ = client.Write([]byte(raw)) }() + return client +} + +// slotsHeld counts the concurrency and tail slots in use. Tests watch it +// instead of probing with requests of their own, which would compete for +// the slot under test. +func slotsHeld(h *harness) int { + h.server.limits.mu.Lock() + defer h.server.limits.mu.Unlock() + n := h.server.limits.total + for _, c := range h.server.limits.active { + n += c + } + return n +} + +func eventually(t *testing.T, within time.Duration, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(within) + for !cond() { + if time.Now().After(deadline) { + t.Fatal(msg) + } + time.Sleep(10 * time.Millisecond) + } +} + +func readResponse(t *testing.T, conn net.Conn) *http.Response { + t.Helper() + got := make(chan *http.Response, 1) + go func() { + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + close(got) + return + } + got <- resp + }() + select { + case resp, ok := <-got: + if !ok { + t.Fatal("reading the response failed") + } + return resp + case <-time.After(3 * time.Second): + t.Fatal("no response") + } + return nil +} + +// atUpstream calls probe each time a request reaches the upstream and +// sends the result on the returned channel. The proxy holds the request's +// slot from before it calls the upstream until the upstream answers, so +// the probe sees it held however late the test goroutine gets scheduled. +func atUpstream(probe func(*harness) int, next http.HandlerFunc) (http.HandlerFunc, *atomic.Pointer[harness], <-chan int) { + var h atomic.Pointer[harness] + seen := make(chan int, 1) + return func(w http.ResponseWriter, r *http.Request) { + seen <- probe(h.Load()) + next(w, r) + }, &h, seen +} + +func receive(t *testing.T, ch <-chan int, msg string) int { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(3 * time.Second): + t.Fatal(msg) + } + return 0 +} + +// TestServerSlowBodyHoldsSlotUntilDeadline sends a POST whose body never +// completes: it must count against the user's concurrency, and the read +// must give up at the request timeout instead of holding the handler. +func TestServerSlowBodyHoldsSlotUntilDeadline(t *testing.T) { + // The slot is only observable while the body is pending, so the + // timeout is long enough for the poll below to see it on a busy runner. + h := newHarness(t, Config{ + RequestTimeout: time.Second, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, nil) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "POST "+logsRoot+"query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=") + + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, + "a request with an unfinished body does not hold a concurrency slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode) + } + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "the slot of an unfinished body was not released") +} + +// TestServerStalledClientReleasesTail holds a tail whose client stops +// reading, once before the headers and once after the first line: the +// slot must come back within a few idle timeouts in both cases. +func TestServerStalledClientReleasesTail(t *testing.T) { + for _, readHeaders := range []bool{false, true} { + lines := make(chan string, 1) + done := make(chan struct{}) + upstream, hp, held := atUpstream(func(h *harness) int { return h.server.limits.openTails() }, tailUpstream(lines, done)) + h := newHarness(t, Config{ + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxTails: 1}, + }, upstream) + hp.Store(h) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "GET "+logsRoot+"tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + if readHeaders { + if resp := readResponse(t, conn); resp.StatusCode != http.StatusOK { + t.Fatalf("tail answered %d", resp.StatusCode) + } + lines <- `{"_msg":"unread"}` + } + if n := receive(t, held, "the tail never reached the store"); n != 1 { + t.Fatalf("open tails while the store answers = %d, want 1", n) + } + eventually(t, 2*time.Second, func() bool { return h.server.limits.openTails() == 0 }, + "a client that stopped reading keeps its tail slot") + } +} + +// TestServerStalledClientReleasesQuery does the same for a non-tail +// response, which must be written by the request deadline. +func TestServerStalledClientReleasesQuery(t *testing.T) { + upstream, hp, held := atUpstream(slotsHeld, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/stream+json") + _, _ = io.WriteString(w, `{"_msg":"hello"}`+"\n") + }) + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, upstream) + hp.Store(h) + pipe := servePipe(t, h.server) + pipe.send(t, "GET "+logsRoot+"query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + if n := receive(t, held, "the stalled request never reached the store"); n != 1 { + t.Fatalf("slots held while the store answers = %d, want 1", n) + } + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "a client that stopped reading keeps its concurrency slot") +} + +// TestServerActiveTailOutlivesWriteDeadline keeps a tail busy for several +// idle timeouts: the write deadline must be renewed, not fixed. +func TestServerActiveTailOutlivesWriteDeadline(t *testing.T) { + lines := make(chan string) + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{TailIdleTimeout: 150 * time.Millisecond}, tailUpstream(lines, done)) + resp := h.do(http.MethodGet, logsRoot+"tail?query=*", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + rd := bufio.NewReader(resp.Body) + for i := range 8 { + lines <- `{"_msg":"line"}` + if _, err := rd.ReadString('\n'); err != nil { + t.Fatalf("line %d: %v", i, err) + } + time.Sleep(75 * time.Millisecond) + } +} + +// TestServerTailUpstreamWithoutHeaders gives up on a store that accepts a +// tail but never answers it, instead of holding the slot until the client +// leaves. +func TestServerTailUpstreamWithoutHeaders(t *testing.T) { + release := make(chan struct{}) + h := newHarness(t, Config{ + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxTails: 1}, + }, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + // After the harness, so it runs before the servers wait for handlers. + t.Cleanup(func() { close(release) }) + got := make(chan int, 1) + go func() { got <- h.status(http.MethodGet, logsRoot+"tail?query=*", "good") }() + select { + case code := <-got: + if code != http.StatusGatewayTimeout { + t.Fatalf("silent store answered %d, want 504", code) + } + case <-time.After(2 * time.Second): + t.Fatal("a tail waits for upstream headers without bound") + } + if n := h.server.limits.openTails(); n != 0 { + t.Fatalf("open tails = %d after the tail ended", n) + } +} + +// TestServerAbortsTruncatedResponse cuts the client connection when the +// store's body breaks off after the headers went out, so a partial answer +// does not end as a clean 200. +func TestServerAbortsTruncatedResponse(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"_msg":"a"}`+"\n") + }) + resp := h.do(http.MethodGet, logsRoot+"query?query=*", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("a truncated store answer reached the client as a complete response") + } +} + +// TestServerUndeliveredBodyReleasesSlot declares a body that never comes +// on requests whose form parsing does not read it: a GET, a POST that is +// not a form, and a GET tail. net/http would drain that body with no +// deadline once the headers go out, so the slot must still come back. +func TestServerUndeliveredBodyReleasesSlot(t *testing.T) { + for name, raw := range map[string]string{ + "get query": "GET " + logsRoot + "query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 10\r\n\r\n", + "post text query": "POST " + logsRoot + "query?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Type: text/plain\r\nContent-Length: 10\r\n\r\n", + "get tail": "GET " + logsRoot + "tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 10\r\n\r\n", + } { + t.Run(name, func(t *testing.T) { + lines := make(chan string, 1) + done := make(chan struct{}) + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + TailIdleTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}, + }, tailUpstream(lines, done)) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + conn := pipe.send(t, raw) + held := func() bool { return slotsHeld(h) > 0 } + eventually(t, time.Second, held, "the request never took a slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("undelivered body answered %d, want 408", resp.StatusCode) + } + eventually(t, time.Second, func() bool { return !held() }, "an undelivered body keeps its slot") + }) + } +} + +func TestServerOversizedBody(t *testing.T) { + h := newHarness(t, Config{}, nil) + for _, ct := range []string{"application/x-www-form-urlencoded", "text/plain"} { + req, _ := http.NewRequest(http.MethodPost, h.proxy.URL+logsRoot+"query?query=*", strings.NewReader("query="+strings.Repeat("x", maxFormBytes))) + req.Header.Set("Authorization", "Bearer good") + req.Header.Set("Content-Type", ct) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Errorf("%s body over the cap answered %d, want 413", ct, resp.StatusCode) + } + } +} + +// TestServerEarlyRefusalWithUnreadBody refuses requests before the body is +// read: a bad token, a full concurrency slot, and a full tail slot. +// net/http drains up to 256 KiB of an unread body before it sends the +// headers, so the answer must not wait on a body that never comes, even +// under a long request timeout. +func TestServerEarlyRefusalWithUnreadBody(t *testing.T) { + for _, timeout := range []time.Duration{200 * time.Millisecond, time.Minute} { + for _, tc := range []struct { + name, raw string + hold string + want int + }{ + {"bad token", "POST " + logsRoot + "query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer bad\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 1000\r\n\r\nquery=", "", http.StatusUnauthorized}, + {"concurrency slot taken", "POST " + logsRoot + "query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 1000\r\n\r\nquery=", "query", http.StatusTooManyRequests}, + {"tail slot taken", "GET " + logsRoot + "tail?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\nContent-Length: 1000\r\n\r\n", "tail", http.StatusTooManyRequests}, + } { + // A held query ends at the request timeout by design, so the + // slot is only reliably taken under the long one. + if tc.hold == "query" && timeout < time.Second { + continue + } + t.Run(fmt.Sprintf("%s/%s", timeout, tc.name), func(t *testing.T) { + lines := make(chan string, 1) + done := make(chan struct{}) + h := newHarness(t, Config{ + RequestTimeout: timeout, + TailIdleTimeout: time.Minute, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}, + }, tailUpstream(lines, done)) + t.Cleanup(func() { close(done) }) + pipe := servePipe(t, h.server) + if tc.hold != "" { + pipe.send(t, "GET "+logsRoot+tc.hold+"?query=* HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the first request never took its slot") + } + start := time.Now() + conn := pipe.send(t, tc.raw) + resp := readResponse(t, conn) + if resp.StatusCode != tc.want || time.Since(start) > time.Second { + t.Fatalf("answered %d after %s, want %d within a second", resp.StatusCode, time.Since(start), tc.want) + } + }) + } + } +} + +// stalledAPIServer answers the TokenReview for "good" unless stallTokens +// is set, and never answers a SubjectAccessReview: it holds each stalled +// call until the caller gives up or the test ends. +func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface { + t.Helper() + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/tokenreviews") && !stallTokens { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"apiVersion":"authentication.k8s.io/v1","kind":"TokenReview","status":{"authenticated":true,"user":{"username":"alice"}}}`) + return + } + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(srv.Close) + // After srv, so it runs first and srv.Close does not wait on a stalled call. + t.Cleanup(func() { close(release) }) + return kubernetes.NewForConfigOrDie(&rest.Config{Host: srv.URL}) +} + +// TestServerStalledReviewAnswersInTime holds TokenReview or +// SubjectAccessReview open on a POST whose body is complete but unread: +// net/http does not watch the connection until the body is read, so only +// a bounded review call ends the request. +func TestServerStalledReviewAnswersInTime(t *testing.T) { + for _, tc := range []struct { + name string + stallTokens bool + }{{"token review", true}, {"subject access review", false}} { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) + h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second) + got := make(chan int, 1) + go func() { + resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil) + _ = resp.Body.Close() + got <- resp.StatusCode + }() + select { + case code := <-got: + if code != http.StatusServiceUnavailable { + t.Fatalf("stalled %s answered %d, want 503", tc.name, code) + } + case <-time.After(2 * time.Second): + t.Fatalf("a stalled %s holds the request past ten request timeouts", tc.name) + } + }) + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go new file mode 100644 index 000000000..df3e23bc4 --- /dev/null +++ b/internal/observabilityproxy/server_test.go @@ -0,0 +1,496 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "bufio" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +type upstreamRequest struct { + path string + form url.Values + headers http.Header +} + +type harness struct { + t *testing.T + kube *fakeKube + proxy *httptest.Server + server *Server + mu sync.Mutex + got []upstreamRequest + stores []string + upstream *httptest.Server +} + +type mapStores map[string]string + +func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } + +// newHarness wires the proxy to a fake apiserver that knows tokens "good" +// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only, +// and to an upstream served by handler. +func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { + h := &harness{t: t} + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return a.Namespace == "tenant-a" && a.Resource == "logs" + }) + h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("upstream ParseForm: %v", err) + } + h.mu.Lock() + h.got = append(h.got, upstreamRequest{path: r.URL.Path, form: r.PostForm, headers: r.Header.Clone()}) + h.mu.Unlock() + if handler != nil { + handler(w, r) + return + } + w.Header().Set("Content-Type", "application/stream+json") + _, _ = io.WriteString(w, `{"_msg":"hello"}`+"\n") + })) + t.Cleanup(h.upstream.Close) + cfg.UpstreamURL = func(store string) string { + h.mu.Lock() + h.stores = append(h.stores, store) + h.mu.Unlock() + return h.upstream.URL + } + if cfg.Logs == (LogsLimits{}) { + cfg.Logs = testLimits() + } + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server.now = func() time.Time { return testNow } + h.proxy = httptest.NewServer(h.server) + t.Cleanup(h.proxy.Close) + return h +} + +const logsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/logsql/" + +func (h *harness) do(method, path, token string, body url.Values, headers map[string]string) *http.Response { + h.t.Helper() + var rd io.Reader + if body != nil { + rd = strings.NewReader(body.Encode()) + } + req, err := http.NewRequest(method, h.proxy.URL+path, rd) + if err != nil { + h.t.Fatal(err) + } + if body != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + h.t.Fatal(err) + } + return resp +} + +func (h *harness) status(method, path, token string) int { + h.t.Helper() + resp := h.do(method, path, token, nil, nil) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode +} + +func (h *harness) requests() []upstreamRequest { + h.mu.Lock() + defer h.mu.Unlock() + return append([]upstreamRequest(nil), h.got...) +} + +func TestServerStatusCodes(t *testing.T) { + h := newHarness(t, Config{}, nil) + tests := []struct { + name, method, path, token string + want int + }{ + {"ok", http.MethodGet, logsRoot + "query?query=error", "good", http.StatusOK}, + {"missing token", http.MethodGet, logsRoot + "query", "", http.StatusUnauthorized}, + {"invalid token", http.MethodGet, logsRoot + "query", "bad", http.StatusUnauthorized}, + {"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden}, + {"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound}, + {"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound}, + {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, + {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, + {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, + {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, + {"ignore_global_time_filter", http.MethodGet, logsRoot + "query?query=" + url.QueryEscape("options(ignore_global_time_filter=true) *"), "good", http.StatusBadRequest}, + {"inverted range", http.MethodGet, logsRoot + "query?start=2026-11-04T11:00:00Z&end=2026-11-04T10:00:00Z", "good", http.StatusBadRequest}, + {"dot dot reaching another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/../tenant-a/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"encoded dot dot", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/%2e%2e/tenant-a/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"encoded slash", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a%2Flogs/logs/select/logsql/query", "good", http.StatusBadRequest}, + {"double slash", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs//select/logsql/query", "good", http.StatusBadRequest}, + {"delete", http.MethodDelete, logsRoot + "query", "good", http.StatusMethodNotAllowed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := h.status(tt.method, tt.path, tt.token); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + }) + } + if reqs := h.requests(); len(reqs) != 1 { + t.Fatalf("upstream saw %d requests, want only the allowed one", len(reqs)) + } +} + +func TestServerRebuildsUpstreamRequest(t *testing.T) { + h := newHarness(t, Config{}, nil) + hostile := url.Values{ + "query": {"error"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{kubernetes_namespace_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret; other=1"} + + resp := h.do(http.MethodGet, logsRoot+"query?"+hostile.Encode(), "good", nil, headers) + _ = resp.Body.Close() + resp = h.do(http.MethodPost, logsRoot+"query?extra_filters="+url.QueryEscape(`{x="y"}`), "good", hostile, headers) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != `{"_msg":"hello"}`+"\n" { + t.Fatalf("POST answered %d %q", resp.StatusCode, body) + } + if ct := resp.Header.Get("Content-Type"); ct != "application/stream+json" { + t.Fatalf("Content-Type = %q, want the upstream's", ct) + } + + reqs := h.requests() + if len(reqs) != 2 { + t.Fatalf("upstream saw %d requests", len(reqs)) + } + if reqs[0].form.Encode() != reqs[1].form.Encode() { + t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form) + } + if n := h.kube.sars.Load(); n != 1 { + t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n) + } + for _, r := range reqs { + if r.path != "/select/logsql/query" { + t.Errorf("upstream path = %s", r.path) + } + if got := r.form["extra_stream_filters"]; len(got) != 1 || got[0] != wantScope { + t.Errorf("extra_stream_filters = %q", got) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") { + t.Errorf("client scope parameter forwarded: %v", r.form) + } + if r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("tenant headers = %q/%q, want 0/0", r.headers.Get("AccountID"), r.headers.Get("ProjectID")) + } + if ae := r.headers.Get("Accept-Encoding"); !strings.Contains(ae, "gzip") { + t.Errorf("upstream query Accept-Encoding = %q, want gzip outside tails", ae) + } + for _, k := range []string{"Authorization", "X-Forwarded-For", "Cookie"} { + if r.headers.Get(k) != "" { + t.Errorf("client header %s forwarded", k) + } + } + } + for _, s := range h.stores { + if s != "tenant-a-store" { + t.Errorf("store = %s, want the resolved one", s) + } + } +} + +func TestServerUpstreamErrors(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "cannot parse query", http.StatusBadRequest) + }) + if got := h.status(http.MethodGet, logsRoot+"query?query=|", "good"); got != http.StatusBadRequest { + t.Fatalf("upstream 400 answered as %d", got) + } + h.upstream.Close() + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusBadGateway { + t.Fatalf("unreachable upstream answered %d, want 502", got) + } +} + +func TestServerLimits(t *testing.T) { + release := make(chan struct{}) + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + }) + first := h.do(http.MethodGet, logsRoot+"query", "good", nil, nil) + defer func() { _ = first.Body.Close() }() + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusTooManyRequests { + t.Fatalf("concurrent request answered %d, want 429", got) + } + once.Do(func() { close(release) }) + + h2 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, nil) + if got := h2.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusOK { + t.Fatalf("first request answered %d", got) + } + limited := h2.do(http.MethodGet, logsRoot+"query", "good", nil, nil) + _ = limited.Body.Close() + if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" { + t.Fatalf("rate-limited request answered %d with Retry-After %q, want 429 with 1000", + limited.StatusCode, limited.Header.Get("Retry-After")) + } +} + +// tailUpstream sends one line per value received on lines and holds the +// stream open until done is closed. +func tailUpstream(lines <-chan string, done <-chan struct{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/stream+json") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + for { + select { + case l := <-lines: + _, _ = io.WriteString(w, l+"\n") + w.(http.Flusher).Flush() + case <-done: + return + case <-r.Context().Done(): + return + } + } + } +} + +func TestServerTailFlushesEveryLine(t *testing.T) { + lines := make(chan string) + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{TailIdleTimeout: time.Minute, RequestTimeout: 50 * time.Millisecond}, tailUpstream(lines, done)) + + resp := h.do(http.MethodGet, logsRoot+"tail?query=*&since=2026-11-04T08:00:00Z", "good", nil, nil) + defer func() { _ = resp.Body.Close() }() + rd := bufio.NewReader(resp.Body) + for i, l := range []string{`{"_msg":"one"}`, `{"_msg":"two"}`} { + select { + case lines <- l: + case <-time.After(2 * time.Second): + t.Fatalf("upstream tail closed before line %d", i) + } + got := make(chan string, 1) + go func() { + s, _ := rd.ReadString('\n') + got <- s + }() + select { + case s := <-got: + if s != l+"\n" { + t.Fatalf("line %d = %q", i, s) + } + case <-time.After(2 * time.Second): + t.Fatalf("line %d not flushed to the client", i) + } + // Past the request timeout: a tail must not be cut by it. + time.Sleep(60 * time.Millisecond) + } + if r := h.requests()[0]; r.path != "/select/logsql/tail" || r.form.Get("start_offset") != "900000ms" { + t.Fatalf("tail upstream = %s %v, want start_offset capped at 15m", r.path, r.form) + } +} + +// TestServerTailHeadersBeforeFirstLine mimics vlselect, which holds even the +// response headers of a gzip-negotiated tail until the first line arrives. +func TestServerTailHeadersBeforeFirstLine(t *testing.T) { + done := make(chan struct{}) + h := newHarness(t, Config{TailIdleTimeout: time.Minute}, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + select { + case <-done: + case <-r.Context().Done(): + } + return + } + tailUpstream(nil, done)(w, r) + }) + // Registered after the harness so it runs before the servers close and + // releases a held upstream when the assertion below fails. + t.Cleanup(func() { close(done) }) + + req, err := http.NewRequest(http.MethodGet, h.proxy.URL+logsRoot+"tail?query=*", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer good") + got := make(chan *http.Response, 1) + go func() { + resp, err := http.DefaultClient.Do(req) + if err != nil { + close(got) + return + } + got <- resp + }() + select { + case resp, ok := <-got: + if !ok { + t.Fatal("tail request failed") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + case <-time.After(2 * time.Second): + t.Fatal("tail response headers did not reach the client before the first line") + } + reqs := h.requests() + if len(reqs) != 1 { + t.Fatalf("upstream saw %d requests, want 1", len(reqs)) + } + if ae := reqs[0].headers.Get("Accept-Encoding"); ae != "identity" { + t.Fatalf("upstream tail Accept-Encoding = %q, want identity", ae) + } +} + +func TestServerTailCapsAndIdle(t *testing.T) { + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + h := newHarness(t, Config{ + Limiter: LimiterConfig{MaxTailsPerUser: 1, MaxTails: 2}, + TailIdleTimeout: 500 * time.Millisecond, + }, tailUpstream(make(chan string), done)) + + a := h.do(http.MethodGet, logsRoot+"tail", "good", nil, nil) + defer func() { _ = a.Body.Close() }() + if a.StatusCode != http.StatusOK { + t.Fatalf("first tail answered %d", a.StatusCode) + } + if got := h.status(http.MethodGet, logsRoot+"tail", "good"); got != http.StatusTooManyRequests { + t.Fatalf("second tail of one user answered %d, want 429", got) + } + b := h.do(http.MethodGet, logsRoot+"tail", "bob-token", nil, nil) + defer func() { _ = b.Body.Close() }() + if b.StatusCode != http.StatusOK { + t.Fatalf("bob's tail answered %d", b.StatusCode) + } + if got := h.server.limits.openTails(); got != 2 { + t.Fatalf("open tails = %d, want 2", got) + } + + closed := make(chan error, 1) + go func() { + _, err := io.Copy(io.Discard, a.Body) + closed <- err + }() + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("idle tail not closed") + } + deadline := time.Now().Add(2 * time.Second) + for h.server.limits.openTails() > 1 { + if time.Now().After(deadline) { + t.Fatal("closed tail still holds its slot") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestServerUpstreamDeadline(t *testing.T) { + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + h := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + if got := h.status(http.MethodGet, logsRoot+"query", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("slow upstream answered %d, want 504", got) + } +} + +func TestServerTailLongLine(t *testing.T) { + long := `{"_msg":"` + strings.Repeat("x", 3*tailChunkBytes) + `"}` + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, long+"\n"+`{"_msg":"short"}`+"\n") + }) + resp := h.do(http.MethodGet, logsRoot+"tail", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if string(body) != long+"\n"+`{"_msg":"short"}`+"\n" { + t.Fatalf("long line not streamed intact: got %d bytes", len(body)) + } +} + +// The auth scheme is case-insensitive (RFC 9110, section 11.1), as +// kube-apiserver reads it. +func TestServerBearerSchemeCase(t *testing.T) { + h := newHarness(t, Config{}, nil) + for _, auth := range []string{"Bearer good", "bearer good", "BEARER good"} { + resp := h.do(http.MethodGet, logsRoot+"query", "", nil, map[string]string{"Authorization": auth}) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("Authorization %q answered %d, want 200", auth, resp.StatusCode) + } + } + for _, auth := range []string{"Basic good", "Bearer", "Bearer ", "Bearergood"} { + resp := h.do(http.MethodGet, logsRoot+"query", "", nil, map[string]string{"Authorization": auth}) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("Authorization %q answered %d, want 401", auth, resp.StatusCode) + } + } +} + +// A 429 for concurrency tells the client when to retry, as the rate 429 +// does, so console polls back off instead of retrying at once. +func TestServerConcurrencyRetryAfter(t *testing.T) { + release := make(chan struct{}) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1, MaxTails: 1}}, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + }) + t.Cleanup(func() { close(release) }) + for _, endpoint := range []string{"query", "tail"} { + first := h.do(http.MethodGet, logsRoot+endpoint, "good", nil, nil) + defer func() { _ = first.Body.Close() }() + second := h.do(http.MethodGet, logsRoot+endpoint, "good", nil, nil) + _ = second.Body.Close() + if second.StatusCode != http.StatusTooManyRequests || second.Header.Get("Retry-After") != "1" { + t.Errorf("%s over the limit answered %d with Retry-After %q, want 429 with 1", + endpoint, second.StatusCode, second.Header.Get("Retry-After")) + } + } +} diff --git a/internal/observabilityproxy/store.go b/internal/observabilityproxy/store.go new file mode 100644 index 000000000..f24f0b17b --- /dev/null +++ b/internal/observabilityproxy/store.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/validation" + corev1listers "k8s.io/client-go/listers/core/v1" +) + +// monitoringLabel is the label the tenant chart sets on every tenant +// namespace and the collector routes logs by +// (packages/system/monitoring-agents/templates/_tenant-log-routes.tpl). +const monitoringLabel = "namespace.cozystack.io/monitoring" + +// StoreResolver maps a namespace to the tenant namespace whose store holds +// its logs, by the same rule the collector writes with. +type StoreResolver struct { + namespaces corev1listers.NamespaceLister + synced func() bool + root string +} + +// NewStoreResolver reads namespaces from a cache that synced reports as +// filled. +func NewStoreResolver(namespaces corev1listers.NamespaceLister, synced func() bool, rootTenant string) *StoreResolver { + return &StoreResolver{namespaces: namespaces, synced: synced, root: rootTenant} +} + +// Resolve answers the root tenant for a namespace that no longer exists, +// because its label went with it. Lines it wrote while routed to another +// tenant's store are not found there; reading them needs routing history +// the proxy does not keep. +func (r *StoreResolver) Resolve(namespace string) (string, error) { + // Before the first list every namespace looks gone, which would send + // a labeled one to the root store; readiness alone does not stop a + // client that addresses the pod directly. + if !r.synced() { + return "", errors.New("namespace cache not synced") + } + ns, err := r.namespaces.Get(namespace) + if apierrors.IsNotFound(err) { + return r.root, nil + } + if err != nil { + return "", err + } + target := ns.Labels[monitoringLabel] + if target == "" { + return r.root, nil + } + // The value becomes a DNS name; a dot in it would address a host in + // some other namespace. + if errs := validation.IsDNS1123Label(target); len(errs) > 0 { + return "", fmt.Errorf("namespace %s: invalid %s label %q: %s", namespace, monitoringLabel, target, strings.Join(errs, "; ")) + } + return target, nil +} diff --git a/internal/observabilityproxy/store_test.go b/internal/observabilityproxy/store_test.go new file mode 100644 index 000000000..d21476b98 --- /dev/null +++ b/internal/observabilityproxy/store_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" +) + +// The cases mirror packages/system/monitoring-agents/tests/ +// fluent-bit_tenant_log_routing_test.yaml: reads must resolve to the store +// the collector wrote to. +func TestStoreResolver(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + add := func(name string, labels map[string]string) { + if err := indexer.Add(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}}); err != nil { + t.Fatal(err) + } + } + add("cozy-system", nil) + add("tenant-empty", map[string]string{monitoringLabel: ""}) + for ns, target := range map[string]string{ + "tenant-root": "tenant-root", + "tenant-inherits-root": "tenant-root", + "tenant-a": "tenant-a", + "tenant-a-child": "tenant-a", + "tenant-b": "tenant-b", + } { + add(ns, map[string]string{monitoringLabel: target}) + } + add("tenant-forged", map[string]string{monitoringLabel: "pod.tenant-b"}) + lister := corev1listers.NewNamespaceLister(indexer) + synced := func() bool { return true } + if got, err := NewStoreResolver(lister, synced, "tenant-root").Resolve("tenant-forged"); err == nil { + t.Fatalf("label naming a host outside a namespace resolved to %q", got) + } + + tests := []struct { + root, ns, want string + }{ + {"tenant-root", "cozy-system", "tenant-root"}, + {"tenant-root", "tenant-empty", "tenant-root"}, + {"tenant-root", "tenant-root", "tenant-root"}, + {"tenant-root", "tenant-inherits-root", "tenant-root"}, + {"tenant-root", "tenant-a", "tenant-a"}, + {"tenant-root", "tenant-a-child", "tenant-a"}, + {"tenant-root", "tenant-b", "tenant-b"}, + {"tenant-root", "tenant-gone", "tenant-root"}, + {"tenant-a", "tenant-a", "tenant-a"}, + {"tenant-a", "tenant-a-child", "tenant-a"}, + {"tenant-a", "tenant-root", "tenant-root"}, + {"tenant-a", "tenant-inherits-root", "tenant-root"}, + {"tenant-a", "cozy-system", "tenant-a"}, + } + for _, tt := range tests { + r := NewStoreResolver(lister, synced, tt.root) + got, err := r.Resolve(tt.ns) + if err != nil { + t.Fatalf("root=%s Resolve(%s) error: %v", tt.root, tt.ns, err) + } + if got != tt.want { + t.Errorf("root=%s Resolve(%s) = %s, want %s", tt.root, tt.ns, got, tt.want) + } + } +} + +// TestStoreResolverUnsynced: before the first list, every namespace looks +// absent, and an absent one resolves to the root store; a labeled +// namespace would read the wrong store. +func TestStoreResolverUnsynced(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + r := NewStoreResolver(corev1listers.NewNamespaceLister(indexer), func() bool { return false }, "tenant-root") + if got, err := r.Resolve("tenant-a"); err == nil { + t.Fatalf("unsynced cache resolved tenant-a to %q", got) + } +} diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 571469bfe..bc439d40b 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -237,15 +237,20 @@ spec: - name: REMOTE_STORAGE value: s3 {{- /* + On the useSystemBucket flow every S3 setting but the + release's own S3_PATH below, the platform's + backupStorage.forcePathStyle included, comes + from the clickhouse-backup.yml key the platform projects + into cozy-backups-creds, mounted below as the config file. + clickhouse-backup reloads that file in each API handler + that reaches S3 and kubelet rewrites a mounted Secret in + place, so credentials projected after the Pod started, or + rotated since, reach the running sidecar; environment + would be read only at start and would override the file, + so none of those is set here. + S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3 - addressing: SeaweedFS (the platform default) needs - path-style, while AWS public S3 needs vhost-style in many - regions. On the useSystemBucket flow the value is the - platform's backupStorage.forcePathStyle, which this app - chart cannot read directly, so it is projected into - cozy-backups-creds and consumed via an optional - secretKeyRef (optional: true so the Pod starts before the - first projection). On the legacy paths (own Secret via + addressing. On the legacy paths (own Secret via s3CredentialsSecret.name, or a configured endpoint) we cannot peek inside the Secret at render time, so we keep the historical path-style default — AWS users in that mode @@ -253,12 +258,8 @@ spec: clickhouse-backup config. */}} {{- if .Values.backup.useSystemBucket }} - - name: S3_FORCE_PATH_STYLE - valueFrom: - secretKeyRef: - name: {{ $sidecarSecretName | quote }} - key: forcePathStyle - optional: true + - name: CLICKHOUSE_BACKUP_CONFIG + value: /etc/clickhouse-backup-s3/config.yml {{- else if or (not (eq (default "" .Values.backup.s3CredentialsSecret.name) "")) .Values.backup.endpoint }} - name: S3_FORCE_PATH_STYLE value: "true" @@ -294,44 +295,18 @@ spec: */}} - name: S3_PATH value: {{ $s3Path | quote }} - {{- /* - optional: true in the useSystemBucket path so the Pod - starts even before the platform projector materialises - cozy-backups-creds in this namespace (projection runs at - first BackupJob time; the sidecar surfaces backup errors - only when its HTTP API is exercised). Without this gate - every fresh ClickHouse release with useSystemBucket=true - would hang in CreateContainerConfigError until a - BackupJob fires. Legacy path keeps the strict reference - shape — its Secret is chart-emitted and present at - install time. - */}} - {{- $secretOptional := .Values.backup.useSystemBucket }} + {{- if not .Values.backup.useSystemBucket }} - name: S3_BUCKET - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } - {{- /* - clickhouse-backup's aws-sdk-go-v2 client rejects a scheme-less - S3_ENDPOINT ("was not a valid URI"). On the useSystemBucket flow - the projected cozy-backups-creds.endpoint is a bare host, so - prepend https:// (the platform's CNPG endpoint helper does the - same). Legacy / BYO-Secret endpoints are tenant-supplied with - their own scheme and are consumed verbatim. - */}} - {{- if .Values.backup.useSystemBucket }} - - name: S3_ENDPOINT_HOST - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }}, optional: true } } - - name: S3_ENDPOINT - value: "https://$(S3_ENDPOINT_HOST)" - {{- else }} + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }} } } - name: S3_ENDPOINT valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }} } } - {{- end }} - name: S3_REGION - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }} } } - name: S3_ACCESS_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }} } } - name: S3_SECRET_KEY - valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } } + valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }} } } + {{- end }} {{- if $endpointCAName }} # Trust the mounted private-CA bundle for the S3 endpoint (see # the `backup.endpointCA` field in values.yaml). SSL_CERT_DIR, @@ -355,6 +330,11 @@ spec: - name: backup-s3-ca mountPath: /etc/ssl/cozy-s3-ca readOnly: true +{{- end }} +{{- if .Values.backup.useSystemBucket }} + - name: backup-s3-config + mountPath: /etc/clickhouse-backup-s3 + readOnly: true {{- end }} securityContext: # clickhouse-server stores data under /var/lib/clickhouse with @@ -378,13 +358,14 @@ spec: runAsGroup: 0 seccompProfile: type: RuntimeDefault -{{- if $endpointCAName }} - # Private-CA bundle for the backup sidecar, projected as ca.crt for - # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds - # the whole ClickHouse Pod in ContainerCreating (naming the Secret in - # `kubectl describe`) rather than mounting an empty trust dir and - # failing later inside clickhouse-backup with an opaque x509 error. +{{- if or $endpointCAName .Values.backup.useSystemBucket }} volumes: +{{- if $endpointCAName }} + # Private-CA bundle for the backup sidecar, projected as ca.crt for + # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds + # the whole ClickHouse Pod in ContainerCreating (naming the Secret in + # `kubectl describe`) rather than mounting an empty trust dir and + # failing later inside clickhouse-backup with an opaque x509 error. - name: backup-s3-ca secret: secretName: {{ $endpointCAName | quote }} @@ -392,6 +373,18 @@ spec: - key: {{ $endpointCAKey | quote }} path: ca.crt {{- end }} +{{- if .Values.backup.useSystemBucket }} + # Optional: the platform projects cozy-backups-creds into this + # namespace at the first BackupJob, after the Pod has started. + - name: backup-s3-config + secret: + secretName: {{ $sidecarSecretName | quote }} + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml +{{- end }} +{{- end }} {{- end }} serviceTemplates: - name: svc-template diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 90bcd1066..a01e10ea5 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -2,10 +2,12 @@ suite: backup sidecar S3 wiring (useSystemBucket vs legacy, path-override guard) # Locks in the clickhouse-backup sidecar S3 wiring this PR introduced on the # ClickHouseInstallation PodTemplate: -# - useSystemBucket=true points every S3_* env at the platform-projected -# cozy-backups-creds Secret with optional:true (the Pod starts before the -# first projection) and scopes S3_PATH to /; the -# chart-managed -backup-s3 / -backup Secrets are skipped; +# - useSystemBucket=true mounts the clickhouse-backup.yml key of the +# platform-projected cozy-backups-creds Secret as the sidecar's config +# file with optional:true (the Pod starts before the first projection), +# sets no other S3 setting in the environment, and scopes S3_PATH to +# /; the chart-managed -backup-s3 / +# -backup Secrets are skipped; # - the legacy flow keeps the chart-emitted -backup-s3 Secret and a # strict (non-optional) secretKeyRef; # - s3PathOverride + useSystemBucket fails fast at render time. @@ -25,7 +27,7 @@ set: _cluster: {} tests: - - it: "useSystemBucket=true: sidecar reads cozy-backups-creds (optional) and scopes S3_PATH" + - it: "useSystemBucket=true: sidecar reads its S3 settings from the projected config file and scopes S3_PATH" template: templates/clickhouse.yaml documentSelector: path: kind @@ -45,28 +47,30 @@ tests: - equal: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_PATH")].value value: tenant-test/ch-test + # clickhouse-backup reloads this file in each API handler that reaches + # S3 and kubelet rewrites the mounted Secret in place, so credentials + # projected after the Pod started, or rotated since, reach the running + # sidecar. - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name - value: cozy-backups-creds + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="CLICKHOUSE_BACKUP_CONFIG")].value + value: /etc/clickhouse-backup-s3/config.yml - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.optional + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].mountPath + value: /etc/clickhouse-backup-s3 + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].readOnly value: true - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - # The projected endpoint is a bare host; clickhouse-backup needs a URI, so - # the chart prepends https:// via env-var expansion on this flow. - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.key - value: endpoint - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")].value - value: https://$(S3_ENDPOINT_HOST) + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret + value: + secretName: cozy-backups-creds + # The Pod starts before the first projection. + optional: true + items: + - key: clickhouse-backup.yml + path: config.yml - - it: "useSystemBucket=true: S3_FORCE_PATH_STYLE is sourced from cozy-backups-creds, not hardcoded" + - it: "useSystemBucket=true: no S3 setting is fixed in the sidecar environment" template: templates/clickhouse.yaml documentSelector: path: kind @@ -76,18 +80,21 @@ tests: enabled: true useSystemBucket: true asserts: - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.name - value: cozy-backups-creds - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.key - value: forcePathStyle - - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.optional - value: true + # Environment overrides the config file and is read only at start. - notExists: - # must no longer be the hardcoded literal value: "true" - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].value + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_REGION")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ACCESS_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")] + - notExists: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")] - it: "useSystemBucket=true: chart-managed backup Secrets are skipped" set: @@ -240,7 +247,10 @@ tests: path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="SSL_CERT_DIR")].value value: /etc/ssl/cozy-s3-ca:/etc/ssl/certs - equal: - path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-ca")].secret.secretName + value: my-s3-ca + - equal: + path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret.secretName value: cozy-backups-creds # backup.enabled=false renders no sidecar at all, so the CA must not leak a diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 17e2a2822..2ac5cfc45 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -87,7 +87,7 @@ See the reference for components utilized in this service: - **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk. -- **Air-gapped tenant workers (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos OS disk image fetched over HTTP by CDI and a `.../installer/...` installer reference. Both sources are now overridable — set `talos.imageFactoryURL` (default `https://factory.talos.dev`) to a self-hosted Image Factory, a caching mirror, or an internal HTTP file server, and `talos.installerRepository` (default `factory.talos.dev/installer`) to a mirrored registry. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig and was removed in this release; **in-guest container-image pulls** (pods pulling from registries inside the tenant cluster) still honour no per-tenant `registries.mirrors` override until Phase 2 restores it via `machine.registries.mirrors` knobs. So a strict-egress tenant can now mirror the Talos OS image and installer, but rate-limited in-guest registry pulls remain a Phase 2 follow-up — file an issue if you depend on this and it is not yet landed. +- **Air-gapped tenant workers (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos OS disk image fetched over HTTP by CDI and a `.../installer/...` installer reference. Both sources are now overridable — set `talos.imageFactoryURL` (default `https://factory.talos.dev`) to a self-hosted Image Factory, a caching mirror, or an internal HTTP file server, and `talos.installerRepository` (default `factory.talos.dev/installer`) to a mirrored registry. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig, so the Secret is removed together with the platform `registries` value that fed it. A Package that still sets `registries` keeps installing, the key is ignored. Registry mirrors for tenant workers, **in-guest container-image pulls** (pods pulling from registries inside the tenant cluster) included, are configured per worker pool through `talos.registryMirrors` of the `KubernetesNodes` resource. - **Worker MachineHealthCheck remediation is now enabled by default** (Phase 1 Talos rollover): the worker MHC `maxUnhealthy` moved from a hard-coded `0` (remediation effectively **off**) to a configurable `nodeHealthCheck.maxUnhealthy` defaulting to `"50%"`. CAPI now auto-remediates (deletes and replaces) unhealthy worker Machines while at least 50% of a group stays healthy. `"50%"` deliberately leaves headroom for transient unhealthy nodes during the kubeadm→Talos rollover and slow first boots from `factory.talos.dev`; set `nodeHealthCheck.maxUnhealthy: "0%"` to keep the previous remediation-off behaviour until your fleet is stable on Talos workers. diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index 9e3fd3ee5..662016692 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -45,7 +45,8 @@ spec: cluster: {{ .Release.Name }} tenant: {{ .Release.Namespace }} remoteWrite: - url: http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus + urls: + - http://vminsert-shortterm.{{ $targetTenant }}.svc.{{ $clusterDomain }}:8480/insert/0/prometheus fluent-bit: readinessProbe: httpGet: diff --git a/packages/apps/kubernetes/templates/ingress.yaml b/packages/apps/kubernetes/templates/ingress.yaml index 9b35f9683..9b88a7fce 100644 --- a/packages/apps/kubernetes/templates/ingress.yaml +++ b/packages/apps/kubernetes/templates/ingress.yaml @@ -3,7 +3,7 @@ on an etcd-less tenant the chart collapses to the awaiting-etcd beacon, so no addon controller exists and the Service would select nothing. */}} {{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd (eq .Values.addons.ingressNginx.exposeMethod "Proxied") }} -{{- if .Values.addons.ingressNginx.hosts }} +{{- if and .Values.addons.ingressNginx.hosts $ingress }} --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml b/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml index 9583b16a3..75e9349a8 100644 --- a/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml +++ b/packages/apps/kubernetes/tests/ingress_proxied_gate_test.yaml @@ -79,3 +79,20 @@ tests: asserts: - hasDocuments: count: 0 + + # An empty _namespace.ingress means no ingress-nginx serves this tenant, and + # the API rejects the ingressClassName "" the Ingress would carry. + - it: renders the Service but no Ingress when the tenant has no ingress + set: + _namespace: + ingress: "" + addons: + ingressNginx: + enabled: true + exposeMethod: Proxied + hosts: ["app.example.com"] + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service diff --git a/packages/apps/kubernetes/tests/monitoring_agents_test.yaml b/packages/apps/kubernetes/tests/monitoring_agents_test.yaml new file mode 100644 index 000000000..1dc21181a --- /dev/null +++ b/packages/apps/kubernetes/tests/monitoring_agents_test.yaml @@ -0,0 +1,42 @@ +suite: monitoring-agents HelmRelease values match the monitoring-agents chart + +# The values below are consumed by packages/system/monitoring-agents, which +# reads vmagent.remoteWrite.urls as a list and the tenant label from +# vmagent.externalLabels.tenant. A key the chart does not read is dropped in +# silence, and the chart then falls back to cozy-monitoring defaults that do +# not resolve inside a tenant cluster. Its counterpart is +# packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml. +templates: + - templates/helmreleases/monitoring-agents.yaml + +values: + - values/common.yaml + +release: + name: kubernetes-clusterprod + namespace: tenant-a + +set: + addons.monitoringAgents.enabled: true + _namespace.monitoring: tenant-root + _cluster: + cluster-domain: cozy.local + +tests: + - it: sends metrics to the shortterm vminsert of the monitoring tenant as a urls list + asserts: + - equal: + path: spec.values.vmagent.remoteWrite + value: + urls: + - http://vminsert-shortterm.tenant-root.svc.cozy.local:8480/insert/0/prometheus + + - it: labels series with the tenant that owns the cluster, not the monitoring tenant + asserts: + - equal: + path: spec.values.vmagent.externalLabels + value: + cluster: kubernetes-clusterprod + tenant: tenant-a + - notExists: + path: spec.values.global diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 50c300aff..c508476f4 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -279,13 +279,13 @@ For `sslmode=verify-full` to work, the CA bundle retrieved above must be saved t ### Databases configuration -| Name | Description | Type | Value | -| -------------------------------- | ---------------------------------------- | ------------------- | ----- | -| `databases` | Databases configuration map. | `map[string]object` | `{}` | -| `databases[name].roles` | Roles assigned to users. | `object` | `{}` | -| `databases[name].roles.admin` | List of users with admin privileges. | `[]string` | `[]` | -| `databases[name].roles.readonly` | List of users with read-only privileges. | `[]string` | `[]` | -| `databases[name].extensions` | List of enabled PostgreSQL extensions. | `[]string` | `[]` | +| Name | Description | Type | Value | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- | ------------------- | ----- | +| `databases` | Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. | `map[string]object` | `{}` | +| `databases[name].roles` | Roles assigned to users. | `object` | `{}` | +| `databases[name].roles.admin` | List of users with admin privileges. | `[]string` | `[]` | +| `databases[name].roles.readonly` | List of users with read-only privileges. | `[]string` | `[]` | +| `databases[name].extensions` | List of enabled PostgreSQL extensions. | `[]string` | `[]` | ### Backup parameters diff --git a/packages/apps/postgres/templates/init-script.yaml b/packages/apps/postgres/templates/init-script.yaml index e78279a2d..6306a5fae 100644 --- a/packages/apps/postgres/templates/init-script.yaml +++ b/packages/apps/postgres/templates/init-script.yaml @@ -53,6 +53,9 @@ PostgreSQL identifiers too, capped at 63. */}} {{- range $database, $d := .Values.databases }} + {{- if has $database (list "postgres" "template0" "template1") }} + {{- fail (printf "databases.%s is refused: %q is a PostgreSQL system database. Managing it would hand it, its public schema and the auto-grant event trigger to the \"%s_admin\" role (template1 then passes all of that on to every database created afterwards), and removing the entry later could not drop it. Remove the entry and declare an application database instead; the next upgrade releases a system database a previous version took over, without dropping it" $database $database $database) }} + {{- end }} {{- if or (not (regexMatch "^[-._a-zA-Z0-9]+$" $database)) (gt (len $database) 54) }} {{- fail (printf "invalid database name %q: a database name may contain only letters, digits and the characters -._ and must be at most 54 characters (it is interpolated into superuser SQL and the init-script shell, and it seeds the \"_admin\" and \"_readonly\" PostgreSQL roles, which CREATE ROLE silently truncates past 63 bytes; the next init-job run then cannot drop the truncated role and fails). Changing only its key in values makes the init-job drop the database; to keep the data, a platform administrator first renames it as the CNPG superuser (ALTER DATABASE, and ALTER ROLE for its _admin and _readonly roles), then the key is changed" $database) }} {{- end }} @@ -320,6 +323,67 @@ stringData: EOT done + echo "== release system databases" + {{- /* + Versions that did not refuse postgres, template0 and template1 in + databases took them over like any other entry. DROP DATABASE on them can + only fail (template databases cannot be dropped, and this session is + connected to postgres), which wedged every later upgrade, so they are + handed back instead: the auto-grant trigger goes, since it names a role + about to be dropped and would make every CREATE SCHEMA fail; what the + _admin and _readonly roles own there goes to cozystack:orphaned, + as in the orphaned-roles pass and for the same reason (a tenant's + SECURITY DEFINER function must not end up owned by the superuser), and is + made SECURITY INVOKER (only a role marked "cozystack:managed" is touched: + a role that merely carries one of those names was not created by this + chart and keeps what it owns); the database itself returns to the superuser, + public to pg_database_owner (to the superuser before PostgreSQL 15), each + only when one of those marked roles owns it and before the reassignment + would move it (or when cozystack:orphaned does, after a run that stopped + in between); an owner the chart did not mark keeps what it owns, and + the comment is cleared. The orphaned-roles pass below then drops the + roles; it does not visit template1, hence the per-database pass here. + template0 accepts no connection, and the Job that commented it never got + further, so only the comment is cleared. A database created from + template1 while it was taken over inherited its trigger, naming + template1_admin, and its public schema owned by that role; the trigger + goes, public returns to its default owner if a marked template1_admin + still owns it, and a declared database gets + its trigger back from the grant pass further down. + */}} + SYSTEM_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm' AND d.datname IN ('postgres', 'template0', 'template1')") + echo "system databases to release: $SYSTEM_DBS" + for db in $SYSTEM_DBS; do + if [ "$db" != template0 ]; then + psql -v ON_ERROR_STOP=1 --echo-all --dbname "$db" -v db="$db" <<\EOT + DROP EVENT TRIGGER IF EXISTS trigger_auto_grant; + DROP FUNCTION IF EXISTS auto_grant_schema_privileges(); + SELECT format('ALTER SCHEMA public OWNER TO %s', CASE WHEN current_setting('server_version_num')::int >= 150000 THEN 'pg_database_owner' ELSE 'CURRENT_USER' END) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND (o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\gexec + SELECT format('ALTER DATABASE %I OWNER TO CURRENT_USER', d.datname) FROM pg_database d JOIN pg_roles o ON o.oid = d.datdba WHERE d.datname = :'db' AND (o.rolname = 'cozystack:orphaned' OR o.rolname IN (:'db' || '_admin', :'db' || '_readonly') AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid))\gexec + SELECT format('REASSIGN OWNED BY %I TO "cozystack:orphaned"', r), format('DROP OWNED BY %I', r) FROM unnest(ARRAY[:'db' || '_admin', :'db' || '_readonly']) AS r WHERE EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid JOIN pg_roles u ON u.oid = m.member WHERE g.rolname = 'cozystack:managed' AND u.rolname = r)\gexec + SELECT format('ALTER %s %s SECURITY INVOKER', CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.oid::regprocedure) FROM pg_proc p WHERE p.proowner = '"cozystack:orphaned"'::regrole AND p.prosecdef\gexec + SELECT format('ALTER VIEW %s SET (security_invoker = true)', c.oid::regclass) FROM pg_class c WHERE c.relowner = '"cozystack:orphaned"'::regrole AND c.relkind = 'v' AND current_setting('server_version_num')::int >= 150000 AND NOT coalesce('security_invoker=true' = ANY (c.reloptions), false)\gexec + EOT + fi + if [ "$db" = template1 ]; then + {{- /* + An assignment of its own, so that set -e stops the pass when the lookup + fails: inside the for list its status is discarded, the comment below is + cleared without a clone visited, and no later run selects template1 again. + */}} + CLONE_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate") + for other in $CLONE_DBS; do + psql -v ON_ERROR_STOP=1 --echo-queries --dbname "$other" <<\EOT + SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant', 'DROP FUNCTION auto_grant_schema_privileges()' WHERE position('''template1_admin''' IN coalesce((SELECT prosrc FROM pg_proc WHERE oid = to_regprocedure('auto_grant_schema_privileges()')), '')) > 0\gexec + SELECT format('ALTER SCHEMA public OWNER TO %s', CASE WHEN current_setting('server_version_num')::int >= 150000 THEN 'pg_database_owner' ELSE 'CURRENT_USER' END) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS (SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid)\gexec + EOT + done + fi + psql -v ON_ERROR_STOP=1 --echo-all -v db="$db" <<\EOT + COMMENT ON DATABASE :"db" IS NULL; + EOT + done + echo "== delete databases" {{- /* A restore replaces spec.databases with the Backup's snapshot, but recovery @@ -337,8 +401,8 @@ stringData: RESTORE_MARK={{ with .Values.bootstrap.newServerName }}$(echo {{ b64enc . }} | base64 -d){{ end }} ADOPTED_MARK=$(psql -v ON_ERROR_STOP=1 -t -A --dbname postgres -c "SELECT current_setting('cozystack.adopted_restore', true)") MANAGED_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm'") - DEFINED_DBS="{{ join " " (keys .Values.databases) }}" - DELETE_DBS=$(for db in $MANAGED_DBS; do case " $DEFINED_DBS " in *" $db "*) :;; *) echo $db;; esac; done) + DEFINED_DBS="{{ join " " (keys .Values.databases | sortAlpha) }}" + DELETE_DBS=$(for db in $MANAGED_DBS; do case " postgres template0 template1 $DEFINED_DBS " in *" $db "*) :;; *) echo $db;; esac; done) if [ -n "$RESTORE_MARK" ] && [ "$ADOPTED_MARK" != "$RESTORE_MARK" ]; then echo "databases the restore brought back that values do not declare, released from management: $DELETE_DBS" diff --git a/packages/apps/postgres/tests/init_script_deterministic_test.yaml b/packages/apps/postgres/tests/init_script_deterministic_test.yaml new file mode 100644 index 000000000..25d0facdd --- /dev/null +++ b/packages/apps/postgres/tests/init_script_deterministic_test.yaml @@ -0,0 +1,37 @@ +suite: init-script renders the same for the same values + +# keys returns a map's keys in Go's randomized iteration order, so a release with +# several databases rendered DEFINED_DBS in a different order on each upgrade. The +# init-script Secret then changed on upgrades that changed nothing in the +# databases, and so did the checksum the init Job carries. + +templates: + - templates/init-script.yaml + +release: + name: pg-test + namespace: tenant-test + +set: + _cluster: + scheduling: {} + +tests: + - it: lists the declared databases in a stable order + set: + databases: + hotel: {} + alpha: {} + golf: {} + charlie: {} + foxtrot: {} + bravo: {} + echo: {} + delta: {} + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: 'DEFINED_DBS="alpha bravo charlie delta echo foxtrot golf hotel"' diff --git a/packages/apps/postgres/tests/system_databases_test.yaml b/packages/apps/postgres/tests/system_databases_test.yaml new file mode 100644 index 000000000..e4ff8400f --- /dev/null +++ b/packages/apps/postgres/tests/system_databases_test.yaml @@ -0,0 +1,135 @@ +suite: init-script never takes over or drops a system database + +# Each of postgres, template0 and template1 used to be accepted in databases and +# handled like an application database: owned by _admin, given the +# auto-grant event trigger, and on removal sent to DROP DATABASE, which fails on +# all three (template databases cannot be dropped, and the Job is connected to +# postgres) and wedged every later upgrade. template1 also passed the takeover +# on to every database created afterwards. + +templates: + - templates/init-script.yaml + +release: + name: pg-test + namespace: tenant-test + +set: + _cluster: + scheduling: {} + +tests: + - it: refuses postgres in databases + set: + databases: + postgres: {} + asserts: + - failedTemplate: + errorPattern: "databases.postgres is refused: \"postgres\" is a PostgreSQL system database" + + - it: refuses template0 in databases + set: + databases: + template0: {} + asserts: + - failedTemplate: + errorPattern: "databases.template0 is refused: \"template0\" is a PostgreSQL system database" + + - it: refuses template1 in databases + set: + databases: + template1: {} + asserts: + - failedTemplate: + errorPattern: "databases.template1 is refused: \"template1\" is a PostgreSQL system database" + + - it: still accepts an application database whose name contains a system name + set: + databases: + postgres_app: {} + mytemplate1: {} + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: 'CREATE DATABASE "postgres_app"' + + # A release that a previous version let manage a system database still carries + # its comment. The delete pass must never select it, whatever values say. + - it: never lists a system database for DROP DATABASE + set: + databases: + appdb: {} + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: 'DELETE_DBS=\$\(for db in \$MANAGED_DBS; do case " postgres template0 template1 \$DEFINED_DBS " in \*" \$db "\*\) :;; \*\) echo \$db;; esac; done\)' + + - it: hands a taken-over system database back before the delete pass + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "SYSTEM_DBS=\\$\\(psql -v ON_ERROR_STOP=1 -t -A -c \"SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm' AND d.datname IN \\('postgres', 'template0', 'template1'\\)\"\\)" + - matchRegex: + path: stringData["init.sh"] + pattern: '(?s)== release system databases.*DROP EVENT TRIGGER IF EXISTS trigger_auto_grant;\s+DROP FUNCTION IF EXISTS auto_grant_schema_privileges\(\);.*ALTER SCHEMA public OWNER TO %s.*ALTER DATABASE %I OWNER TO CURRENT_USER.*REASSIGN OWNED BY %I TO "cozystack:orphaned"''.*DROP OWNED BY %I.*COMMENT ON DATABASE :"db" IS NULL;.*== delete databases' + + # A tenant holding _admin may have created a SECURITY DEFINER function + # there; handed to the superuser it would run with superuser rights for + # anyone PUBLIC can execute it. It goes to cozystack:orphaned and is made + # SECURITY INVOKER, as in the orphaned-roles pass. + - it: never hands a released system database's objects to the superuser + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - notMatchRegex: + path: stringData["init.sh"] + pattern: 'REASSIGN OWNED BY %I TO CURRENT_USER' + - matchRegex: + path: stringData["init.sh"] + pattern: '(?s)== release system databases.*ALTER %s %s SECURITY INVOKER.*ALTER VIEW %s SET \(security_invoker = true\).*== delete databases' + + # A database created from template1 while it was taken over inherited the + # trigger naming template1_admin, which the orphaned-roles pass then drops; + # CREATE SCHEMA would fail there from that point on. + - it: drops the template1 trigger inherited by other databases + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)== release system databases.*if \\[ \"\\$db\" = template1 \\]; then.*SELECT 'DROP EVENT TRIGGER IF EXISTS trigger_auto_grant', 'DROP FUNCTION auto_grant_schema_privileges\\(\\)' WHERE position\\('''template1_admin''' IN .*== delete databases" + + # That clone's public schema also came from template1, owned by + # template1_admin; left to the orphaned-roles pass it would end up owned by + # cozystack:orphaned and the database owner would lose CREATE on it. + - it: gives an inherited public schema back to its default owner + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)if \\[ \"\\$db\" = template1 \\]; then.*SELECT format\\('ALTER SCHEMA public OWNER TO %s', .*\\) FROM pg_namespace n JOIN pg_roles o ON o.oid = n.nspowner WHERE n.nspname = 'public' AND o.rolname = 'template1_admin' AND EXISTS \\(SELECT FROM pg_auth_members m JOIN pg_roles g ON g.oid = m.roleid WHERE g.rolname = 'cozystack:managed' AND m.member = o.oid\\).*== delete databases" + + # template0 accepts no connection: only postgres and template1 are connected + # to and released, template0 just has its comment cleared. The behaviour is + # run against a fake psql in hack/postgres-release-system-databases.bats. + - it: connects to every released system database but template0 + documentSelector: + path: metadata.name + value: pg-test-init-script + asserts: + - matchRegex: + path: stringData["init.sh"] + pattern: "(?s)for db in \\$SYSTEM_DBS; do\\s+if \\[ \"\\$db\" != template0 \\]; then\\s+psql -v ON_ERROR_STOP=1 --echo-all --dbname \"\\$db\" -v db=\"\\$db\"" diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 91747ba24..4d544ff6b 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -313,7 +313,7 @@ } }, "databases": { - "description": "Databases configuration map.", + "description": "Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.", "type": "object", "default": {}, "additionalProperties": { diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 83b658a64..fb4a67b35 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -287,7 +287,7 @@ users: {} ## @field {DatabaseRoles} [roles] - Roles assigned to users. ## @field {[]string} [extensions] - List of enabled PostgreSQL extensions. -## @param {map[string]Database} databases - Databases configuration map. +## @param {map[string]Database} databases - Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused. databases: {} ## Example: ## databases: diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md index b1ba98d7c..84cb04930 100644 --- a/packages/apps/redis/README.md +++ b/packages/apps/redis/README.md @@ -17,7 +17,7 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc | Name | Description | Type | Value | | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | --------- | -| `replicas` | Number of Redis replicas. | `int` | `2` | +| `replicas` | Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to. | `int` | `2` | | `resources` | Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied. | `object` | `{}` | | `resources.cpu` | CPU available to each replica. | `quantity` | `""` | | `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | @@ -48,13 +48,13 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc ### Sentinel parameters -| Name | Description | Type | Value | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----- | -| `sentinel` | Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. | `object` | `{}` | -| `sentinel.resources` | Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`. | `object` | `{}` | -| `sentinel.resources.cpu` | CPU available to each replica. | `quantity` | `""` | -| `sentinel.resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | -| `sentinel.resourcesPreset` | Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`. | `string` | `{}` | +| Name | Description | Type | Value | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----- | +| `sentinel` | Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. | `object` | `{}` | +| `sentinel.resources` | Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`. | `object` | `{}` | +| `sentinel.resources.cpu` | CPU available to each replica. | `quantity` | `""` | +| `sentinel.resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | +| `sentinel.resourcesPreset` | Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`. | `string` | `{}` | ## Parameter examples and reference @@ -79,7 +79,7 @@ See [Resource management](https://cozystack.io/docs/v1/guides/resource-managemen ### sentinel -`sentinel.resources` and `sentinel.resourcesPreset` size the three Sentinel pods separately from the Redis pods. A Sentinel holds only its configuration and the failover state, so it needs a small fraction of what a Redis replica holding the dataset does. When the `sentinel` block sets neither, the Sentinel pods are sized like the Redis pods, which is how every instance was sized before these parameters existed. +`sentinel.resources` and `sentinel.resourcesPreset` size the Sentinel pods separately from the Redis pods. A Sentinel holds only its configuration and the failover state, so it needs a small fraction of what a Redis replica holding the dataset does. When the `sentinel` block sets neither, the Sentinel pods are sized like the Redis pods, which is how every instance was sized before these parameters existed. ```yaml resourcesPreset: u1.small diff --git a/packages/apps/redis/templates/_resources.tpl b/packages/apps/redis/templates/_resources.tpl index 6539c99a1..45399c958 100644 --- a/packages/apps/redis/templates/_resources.tpl +++ b/packages/apps/redis/templates/_resources.tpl @@ -47,3 +47,7 @@ These presets are for basic testing and not meant to be used in production {{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}} {{- end -}} {{- end -}} + +{{- define "redis.sentinelReplicas" -}} +{{- ternary 1 3 (eq (int .Values.replicas) 1) -}} +{{- end -}} diff --git a/packages/apps/redis/templates/hooks/sentinel-pdb.yaml b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml new file mode 100644 index 000000000..1649caa79 --- /dev/null +++ b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml @@ -0,0 +1,122 @@ +{{- /* + An instance scaled down to one Redis pod keeps the Sentinel PDB the operator + created while it ran three Sentinels: setting disablePodDisruptionBudget only + stops the operator from ensuring it, never deletes it, and minAvailable 1 on + a single Sentinel refuses every eviction. A fresh install with one replica + never gets that PDB, so this runs on upgrade only. +*/}} +{{- if and .Release.IsUpgrade (eq (include "redis.sentinelReplicas" . | int) 1) }} +{{- $name := printf "%s-sentinel-pdb" .Release.Name }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + resourceNames: ["rfs-{{ .Release.Name }}"] + verbs: ["get", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + {{- /* The API server copies a Job's name into its pods' job-name label, + which takes 63 characters at most, while a release name may take 53. + The objects above have no such limit and keep the full name. */}} + name: {{ $name | trunc 63 | trimSuffix "-" }} + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + ttlSecondsAfterFinished: 3600 + # Bounds the kubectl call too: a --request-timeout would make kubectl skip + # its in-cluster credentials (hack/check-hook-request-timeout.bats). + activeDeadlineSeconds: 120 + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + policy.cozystack.io/allow-to-apiserver: "true" + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + terminationGracePeriodSeconds: 10 + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: delete-pdb + image: '{{ include "cozy-lib.image" (list "clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c" $) }}' + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + env: + - name: HOME + value: /tmp + command: + - kubectl + - --namespace={{ .Release.Namespace }} + - delete + - poddisruptionbudget + - rfs-{{ .Release.Name }} + - --ignore-not-found=true + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +{{- end }} diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index 27c7e9047..05a96e3cc 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -9,6 +9,16 @@ top-level resources falls back to resourcesPreset. */}} {{- $sentinel := .Values.sentinel | default dict -}} +{{- /* + A lone Redis pod has no replica to promote, so three Sentinels would only + watch it. One Sentinel still lets the operator find the master (its quorum is + replicas/2+1, so 1). The operator gives the Sentinel Deployment a PDB with + minAvailable 1 whenever Redis has two replicas or fewer, which on a single + Sentinel would block every node drain, so that PDB is dropped with it; the + operator does not delete a PDB it stops managing, hence + hooks/sentinel-pdb.yaml. +*/}} +{{- $sentinelReplicas := include "redis.sentinelReplicas" . | int -}} {{- $sentinelPreset := $sentinel.resourcesPreset | default .Values.resourcesPreset -}} {{- $sentinelResources := .Values.resources -}} {{- if or $sentinel.resources $sentinel.resourcesPreset -}} @@ -65,7 +75,10 @@ spec: # pods they keep no RDB, only a sentinel.conf the init container rewrites # on every start, so a chart rollback to an older engine is safe here. image: {{ $redisImage | quote }} - replicas: 3 + replicas: {{ $sentinelReplicas }} + {{- if eq $sentinelReplicas 1 }} + disablePodDisruptionBudget: true + {{- end }} resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $sentinelPreset $sentinelResources $) | nindent 6 }} redis: image: {{ $redisImage | quote }} @@ -152,8 +165,8 @@ metadata: labels: workloads.cozystack.io/resource-preset: {{ $sentinelPreset | quote }} spec: - minReplicas: 2 - replicas: 3 + minReplicas: {{ ternary 1 2 (eq $sentinelReplicas 1) }} + replicas: {{ $sentinelReplicas }} kind: redis type: sentinel selector: diff --git a/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml new file mode 100644 index 000000000..03f3da02d --- /dev/null +++ b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml @@ -0,0 +1,154 @@ +suite: stale Sentinel PDB removed on scale-down to one Redis pod + +# The operator only stops ensuring the Sentinel PDB once +# disablePodDisruptionBudget is set; it never deletes the one it created while +# three Sentinels ran. Left in place, its minAvailable 1 refuses every eviction +# of the single Sentinel and blocks node drains. + +templates: + - templates/hooks/sentinel-pdb.yaml + +release: + name: my-redis + namespace: tenant-test + +set: + _cluster: + cluster-domain: cluster.local + _namespace: + host: example.com + +tests: + - it: deletes the operator's Sentinel PDB on an upgrade to one replica + release: + upgrade: true + set: + replicas: 1 + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: post-upgrade + - equal: + path: spec.template.spec.containers[0].command + value: + - kubectl + - --namespace=tenant-test + - delete + - poddisruptionbudget + - rfs-my-redis + - --ignore-not-found=true + - notMatchRegex: + path: spec.template.spec.containers[0].command[4] + pattern: request-timeout + + - it: lets the hook delete only that PDB + release: + upgrade: true + set: + replicas: 1 + documentSelector: + path: kind + value: Role + asserts: + - equal: + path: rules + value: + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + resourceNames: ["rfs-my-redis"] + verbs: ["get", "delete"] + + - it: renders nothing on install, where no such PDB exists + set: + replicas: 1 + asserts: + - hasDocuments: + count: 0 + + - it: renders nothing while three Sentinels run + release: + upgrade: true + set: + replicas: 2 + asserts: + - hasDocuments: + count: 0 + + # The operator runs three Redis pods for a zero or negative count, so their + # Sentinel PDB stays. + - it: renders nothing for a zero replica count + release: + upgrade: true + set: + replicas: 0 + asserts: + - hasDocuments: + count: 0 + + - it: renders nothing for a negative replica count + release: + upgrade: true + set: + replicas: -1 + asserts: + - hasDocuments: + count: 0 + + # The pods' job-name label takes 63 characters at most; a 53-character + # release name plus -sentinel-pdb would pass it and Kubernetes would refuse + # the Job, leaving the stale PDB in place. + - it: keeps the Job name within 63 characters for the longest release name + release: + name: redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + upgrade: true + set: + replicas: 1 + documentSelector: + path: kind + value: Job + asserts: + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9-]{1,63}$ + - matchRegex: + path: metadata.name + pattern: '[a-z0-9]$' + # The full release name stays in the Job name, so the hooks of two + # releases in one namespace never replace each other's Job. + - matchRegex: + path: metadata.name + pattern: ^redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa- + - equal: + path: spec.template.spec.containers[0].command[4] + value: rfs-redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + - it: lets the hook delete the Sentinel PDB of the longest release name + release: + name: redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + upgrade: true + set: + replicas: 1 + documentSelector: + path: kind + value: Role + asserts: + - equal: + path: rules[0].resourceNames + value: + - rfs-redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + - it: keeps the full Job name for a short release name + release: + upgrade: true + set: + replicas: 1 + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: my-redis-sentinel-pdb diff --git a/packages/apps/redis/tests/sentinel_replicas_test.yaml b/packages/apps/redis/tests/sentinel_replicas_test.yaml new file mode 100644 index 000000000..fc60d52b0 --- /dev/null +++ b/packages/apps/redis/tests/sentinel_replicas_test.yaml @@ -0,0 +1,131 @@ +suite: Sentinel count follows the Redis replicas + +release: + name: my-redis + namespace: tenant-test + +set: + _cluster: + cluster-domain: cluster.local + _namespace: + host: example.com + +templates: + - templates/redisfailover.yaml + +tests: + - it: "runs a single Sentinel for a single Redis pod" + set: + replicas: 1 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 1 + - equal: + path: spec.redis.replicas + value: 1 + + # The operator's Sentinel PDB is minAvailable 1 at two Redis replicas or + # fewer; on one Sentinel it would refuse every eviction and block node drains. + - it: "drops the Sentinel PDB when only one Sentinel runs" + set: + replicas: 1 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.disablePodDisruptionBudget + value: true + + - it: "keeps three Sentinels and their PDB from two Redis pods up" + set: + replicas: 2 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 3 + - notExists: + path: spec.sentinel.disablePodDisruptionBudget + + # The operator turns a zero or negative count into three Redis pods, so these + # keep three Sentinels and their PDB rather than the single-pod case. + - it: "keeps three Sentinels and their PDB for a zero replica count" + set: + replicas: 0 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 3 + - notExists: + path: spec.sentinel.disablePodDisruptionBudget + + - it: "keeps three Sentinels and their PDB for a negative replica count" + set: + replicas: -1 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 3 + - notExists: + path: spec.sentinel.disablePodDisruptionBudget + + - it: "keeps three Sentinels for three Redis pods" + set: + replicas: 3 + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 3 + + - it: "keeps three Sentinels at the chart default" + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.sentinel.replicas + value: 3 + + - it: "reports one expected Sentinel pod to the workload monitor" + set: + replicas: 1 + documentSelector: + path: metadata.name + value: my-redis-sentinel + asserts: + - equal: + path: spec.replicas + value: 1 + - equal: + path: spec.minReplicas + value: 1 + + - it: "reports three expected Sentinel pods to the workload monitor" + set: + replicas: 2 + documentSelector: + path: metadata.name + value: my-redis-sentinel + asserts: + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.minReplicas + value: 2 diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json index 4bf3e0964..1a301615f 100644 --- a/packages/apps/redis/values.schema.json +++ b/packages/apps/redis/values.schema.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "replicas": { - "description": "Number of Redis replicas.", + "description": "Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.", "type": "integer", "default": 2 }, @@ -172,7 +172,7 @@ "minimum": 0 }, "sentinel": { - "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.", + "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.", "type": "object", "default": {}, "properties": { diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml index 2a580c8e6..7917ca34d 100644 --- a/packages/apps/redis/values.yaml +++ b/packages/apps/redis/values.yaml @@ -55,7 +55,7 @@ ## @value xlarge ## @value 2xlarge -## @param {int} replicas - Number of Redis replicas. +## @param {int} replicas - Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to. replicas: 2 ## @param {Resources} [resources] - Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied. @@ -127,5 +127,5 @@ authEnabled: true ## @field {Resources} [resources] - Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`. ## @field {ResourcesPreset} [resourcesPreset] - Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`. -## @param {Sentinel} [sentinel] - Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. +## @param {Sentinel} [sentinel] - Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. sentinel: {} diff --git a/packages/core/installer/templates/cozystack-operator.yaml b/packages/core/installer/templates/cozystack-operator.yaml index 6f2cc6880..ef2685926 100644 --- a/packages/core/installer/templates/cozystack-operator.yaml +++ b/packages/core/installer/templates/cozystack-operator.yaml @@ -200,6 +200,9 @@ spec: {{- if .Values.cozystackOperator.platformSourceSecret }} - --platform-source-secret={{ .Values.cozystackOperator.platformSourceSecret }} {{- end }} + {{- if .Values.cozystackOperator.platformSourceCertSecret }} + - --platform-source-cert-secret={{ .Values.cozystackOperator.platformSourceCertSecret }} + {{- end }} {{- $hasEndpointEnv := or (eq .Values.cozystackOperator.variant "talos") (eq .Values.cozystackOperator.variant "generic") }} {{- if or .Values.cozystackOperator.platformVersion $hasEndpointEnv }} env: diff --git a/packages/core/installer/tests/platform_source_test.yaml b/packages/core/installer/tests/platform_source_test.yaml new file mode 100644 index 000000000..a925754c1 --- /dev/null +++ b/packages/core/installer/tests/platform_source_test.yaml @@ -0,0 +1,67 @@ +suite: cozystack-operator platform source arguments + +# Each flag must follow its own value: a pull-through cache behind a private CA +# often needs the CA without credentials, so a cert flag keyed on the wrong +# value would bring back the TLS failure while every other case stays green. + +templates: + - templates/cozystack-operator.yaml + +tests: + - it: passes platformSourceCertSecret alone, without credentials + set: + cozystackOperator.platformSourceCertSecret: registry-ca + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --platform-source-cert-secret=registry-ca + documentSelector: + path: kind + value: Deployment + - notExists: + path: spec.template.spec.containers[0].args[?(@ =~ /^--platform-source-secret=/)] + documentSelector: + path: kind + value: Deployment + + - it: passes platformSourceSecret alone, without the cert flag + set: + cozystackOperator.platformSourceSecret: registry-creds + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --platform-source-secret=registry-creds + documentSelector: + path: kind + value: Deployment + - notExists: + path: spec.template.spec.containers[0].args[?(@ =~ /^--platform-source-cert-secret=/)] + documentSelector: + path: kind + value: Deployment + + - it: passes both secrets together + set: + cozystackOperator.platformSourceSecret: registry-creds + cozystackOperator.platformSourceCertSecret: registry-ca + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --platform-source-cert-secret=registry-ca + documentSelector: + path: kind + value: Deployment + - contains: + path: spec.template.spec.containers[0].args + content: --platform-source-secret=registry-creds + documentSelector: + path: kind + value: Deployment + + - it: omits the cert secret flag by default + asserts: + - notExists: + path: spec.template.spec.containers[0].args[?(@ =~ /^--platform-source-cert-secret=/)] + documentSelector: + path: kind + value: Deployment diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 43639ab5c..55f19c0d9 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -63,6 +63,16 @@ cozystackOperator: # otherwise source-controller will report a failed condition until it is created. # See FluxCD source-controller docs for the exact key layout per scheme. platformSourceSecret: '' + # Name of the Secret in cozy-system namespace with TLS material for an OCI + # platform source: `ca.crt` for a registry whose certificate is issued by a + # private CA, and/or `tls.crt`+`tls.key` for client certificate auth. A + # CA-only Secret must be type Opaque: kubernetes.io/tls requires `tls.crt` + # and `tls.key`. When non-empty, sets spec.certSecretRef on the + # generated OCIRepository; the operator rewrites that object on every start, + # so a certSecretRef patched onto it by hand does not survive a restart. + # The cozystack-packages source copies the spec, so it inherits the CA too. + # OCI only: for a Git source put `ca.crt` into the platformSourceSecret Secret. + platformSourceCertSecret: '' # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) diff --git a/packages/core/platform/sources/observability-proxy.yaml b/packages/core/platform/sources/observability-proxy.yaml new file mode 100644 index 000000000..d9e9ec14b --- /dev/null +++ b/packages/core/platform/sources/observability-proxy.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: cozystack.io/v1alpha1 +kind: PackageSource +metadata: + name: cozystack.observability-proxy +spec: + sourceRef: + kind: OCIRepository + name: cozystack-packages + namespace: cozy-system + path: / + variants: + - name: default + dependsOn: + - cozystack.networking + components: + - name: observability-proxy + path: system/observability-proxy + install: + namespace: cozy-observability-proxy + releaseName: observability-proxy diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 030cfe8b2..db6c7109b 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -33,6 +33,14 @@ stringData: opensearch-operator: manager: dnsBase: {{ .Values.networking.clusterDomain | quote }} + {{- /* + The operator resolves its own MariaDB pods, and the webhook certificate + gets its SAN, from CLUSTER_NAME, which the vendored subchart reads from + clusterName. Same subchart-key route as above; a package rendered + without this Secret falls back to the literal in its values.yaml. + */}} + mariadb-operator: + clusterName: {{ .Values.networking.clusterDomain | quote }} _cluster: root-host: {{ $rootHost | quote }} bundle-name: {{ .Values.bundles.system.variant | quote }} @@ -253,6 +261,10 @@ stringData: null/omitted in custom values. Missing key -> the true default, matching the chart values.yaml. */}} monitoring-enabled: {{ dig "rootEnabled" true (.Values.monitoring | default dict) | quote }} + {{- /* Whether the root tenant runs its own ingress-nginx + (Tenant/root spec.ingress). `dig`, as for monitoring-enabled + above, so an explicit false survives. */}} + ingress-enabled: {{ dig "rootEnabled" true (.Values.ingress | default dict) | quote }} {{- /* The rate each shared-central tracing tenant's collector holds it to. Only written when set: every release in every tenant reads this Secret, and a key appearing would upgrade all of them. */}} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index b175714a2..906c8d95f 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -344,6 +344,7 @@ variants and opt-in on the slim ones. */}} {{include "cozystack.platform.package.full.default" (list "cozystack.metrics-server" $) }} {{- $monitoringAgentsComponents := dict "monitoring-agents" (dict "values" (dict "global" (dict "target" "tenant-root"))) -}} {{include "cozystack.platform.package.full" (list "cozystack.monitoring-agents" "default" $ $monitoringAgentsComponents) }} +{{include "cozystack.platform.package.full.default" (list "cozystack.observability-proxy" $) }} {{include "cozystack.platform.package.full.default" (list "cozystack.goldpinger" $) }} {{include "cozystack.platform.package.default" (list "cozystack.prometheus-operator-crds" $) }} {{include "cozystack.platform.package.full.default" (list "cozystack.grafana-operator" $) }} diff --git a/packages/core/platform/templates/containerd-registry-secret.yaml b/packages/core/platform/templates/containerd-registry-secret.yaml deleted file mode 100644 index 6e9fb6408..000000000 --- a/packages/core/platform/templates/containerd-registry-secret.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if .Values.registries.mirrors }} -apiVersion: v1 -kind: Secret -metadata: - name: patch-containerd - namespace: cozy-system -type: Opaque -stringData: -{{- range $registry, $mirror := .Values.registries.mirrors }} -{{- if $mirror.endpoints }} - {{ $registry }}.toml: | - server = "https://{{ $registry }}" -{{- range $endpoint := $mirror.endpoints }} - [host."{{ $endpoint }}"] - capabilities = ["pull", "resolve"] -{{- $endpointConfig := index $.Values.registries.config $endpoint }} -{{- if $endpointConfig }} -{{- if $endpointConfig.tls }} -{{- if $endpointConfig.tls.insecureSkipVerify }} - skip_verify = true -{{- end }} -{{- end }} -{{- if $endpointConfig.auth }} - [host."{{ $endpoint }}".auth] - username = "{{ $endpointConfig.auth.username }}" - password = "{{ $endpointConfig.auth.password }}" -{{- end }} -{{- else }} - skip_verify = true -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - diff --git a/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml b/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml new file mode 100644 index 000000000..68f6a8168 --- /dev/null +++ b/packages/core/platform/tests/apps_ingress_enabled_wiring_test.yaml @@ -0,0 +1,35 @@ +suite: apps.yaml ingress.rootEnabled → _cluster wiring +templates: + - templates/apps.yaml + +release: + name: cozy-platform + namespace: cozy-system + +# ingress.rootEnabled records whether the root tenant runs its own +# ingress-nginx (Tenant/root spec.ingress). It reaches cozystack-basics through +# _cluster.ingress-enabled in the cozystack-values Secret, where it decides +# whether tenant-root's _namespace.ingress names tenant-root or stays empty. + +tests: + - it: writes ingress-enabled true by default + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"true"' + + - it: writes ingress-enabled false when ingress.rootEnabled is disabled + set: + ingress.rootEnabled: false + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"false"' + + - it: falls back to true (no panic) when ingress is null + set: + ingress: null + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s*ingress-enabled:\s*"true"' diff --git a/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml b/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml new file mode 100644 index 000000000..6cd9426ba --- /dev/null +++ b/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml @@ -0,0 +1,33 @@ +suite: apps.yaml networking.clusterDomain reaches the mariadb-operator cluster name + +templates: + - templates/apps.yaml + +release: + name: cozy-platform + namespace: cozy-system + +# The operator dials ..svc. to set up replication and +# to reconcile User, Database and Grant, and the vendored subchart fills +# CLUSTER_NAME from clusterName. A cluster whose domain is not cozy.local got +# NXDOMAIN on every such lookup while the literal was pinned in the package. +# Every pattern anchors clusterName to its mariadb-operator parent, so a sibling +# block carrying the same domain cannot satisfy the assertion on its own. + +tests: + - it: carries the cozy.local default under the subchart key + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^mariadb-operator:\n clusterName: "cozy\.local"$' + + - it: tracks a non-default networking.clusterDomain + set: + networking.clusterDomain: example.test + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^mariadb-operator:\n clusterName: "example\.test"$' + - notMatchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^mariadb-operator:\n clusterName: "cozy\.local"$' diff --git a/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml b/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml index 221aeeb27..6758b1b42 100644 --- a/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml +++ b/packages/core/platform/tests/fixtures/selective-install-postgres-values.yaml @@ -42,6 +42,7 @@ bundles: - cozystack.monitoring - cozystack.nats-application - cozystack.nfs-driver + - cozystack.observability-proxy - cozystack.openbao-application - cozystack.opensearch-application - cozystack.opensearch-operator diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml new file mode 100644 index 000000000..9e6ec583d --- /dev/null +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -0,0 +1,69 @@ +suite: observability-proxy package wiring +# The proxy reads the tenant log and metrics stores, which the full variants +# run by default and the slim ones only on request, so it follows +# monitoring-agents: on by default in the full variants, opt-in on the slim. +templates: + - templates/sources.yaml + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: installs into cozy-observability-proxy once networking is up + template: templates/sources.yaml + documentSelector: + path: metadata.name + value: cozystack.observability-proxy + asserts: + - equal: + path: kind + value: PackageSource + - equal: + path: spec.variants[0].dependsOn + value: + - cozystack.networking + - equal: + path: spec.variants[0].components + value: + - name: observability-proxy + path: system/observability-proxy + install: + namespace: cozy-observability-proxy + releaseName: observability-proxy + + - it: emits the package on a full variant + template: templates/bundles/system.yaml + values: + - ../values-isp-full.yaml + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true + + - it: leaves the package out of a slim variant by default + template: templates/bundles/system.yaml + values: + - ../values-isp-slim.yaml + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true + not: true + + - it: emits the package on a slim variant that lists it in enabledPackages + template: templates/bundles/system.yaml + values: + - ../values-isp-slim.yaml + set: + bundles.enabledPackages: + - cozystack.observability-proxy + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.observability-proxy + any: true diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2ce252ed..f2e35b5e5 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -669,6 +669,19 @@ gateway: - cozy-monitoring - cozy-linstor-gui - default +# Platform ingress configuration. +ingress: + # Whether the root tenant (tenant-root) runs its own ingress-nginx, i.e. + # Tenant/root spec.ingress=true. It decides whether tenant-root's + # _namespace.ingress names tenant-root or stays empty, which is what + # applications there (SeaweedFS waiting on ingress-nginx-system, the + # ingressClassName of their Ingresses) and every tenant inheriting from the + # root read. Nothing in the charts sets Tenant/root spec.ingress, so keep + # this in sync with it by hand; changing it re-renders cozystack-basics and + # the releases reading the cozystack-values Secret. The default is true only + # to keep the value existing clusters already render. + rootEnabled: true + # Platform monitoring configuration. monitoring: # Whether the root tenant (tenant-root) hosts the platform @@ -768,37 +781,6 @@ scheduling: # loginTheme: acme # loginThemeImage: registry.example.com/acme-keycloak-theme:v1 branding: {} -# Container registry mirrors configuration -# -# Example: -# registries: -# mirrors: -# docker.io: -# endpoints: -# - http://10.0.0.1:8082 -# ghcr.io: -# endpoints: -# - http://10.0.0.1:8083 -# gcr.io: -# endpoints: -# - http://10.0.0.1:8084 -# registry.k8s.io: -# endpoints: -# - http://10.0.0.1:8085 -# quay.io: -# endpoints: -# - http://10.0.0.1:8086 -# cr.fluentbit.io: -# endpoints: -# - http://10.0.0.1:8087 -# docker-registry3.mariadb.com: -# endpoints: -# - http://10.0.0.1:8088 -# config: -# "10.0.0.1:8082": -# tls: -# insecureSkipVerify: true -registries: {} # Resource allocation ratios resources: cpuAllocationRatio: 10 diff --git a/packages/extra/seaweedfs/README.md b/packages/extra/seaweedfs/README.md index e269992bf..0540db4af 100644 --- a/packages/extra/seaweedfs/README.md +++ b/packages/extra/seaweedfs/README.md @@ -78,4 +78,7 @@ | `s3.resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | | `s3.resources.memory` | Amount of memory allocated. | `quantity` | `""` | | `s3.resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `t1.small` | +| `lifecycle` | S3 bucket lifecycle configuration. | `object` | `{}` | +| `lifecycle.enabled` | Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new. | `bool` | `false` | +| `lifecycle.storageClass` | StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again. | `string` | `""` | diff --git a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag index 49f252950..bf4084277 100644 --- a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag +++ b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag @@ -1 +1 @@ -ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 +ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 diff --git a/packages/extra/seaweedfs/templates/admin-credentials.yaml b/packages/extra/seaweedfs/templates/admin-credentials.yaml new file mode 100644 index 000000000..4b9f6dd33 --- /dev/null +++ b/packages/extra/seaweedfs/templates/admin-credentials.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.lifecycle.enabled (ne .Values.topology "Client") }} +{{- $name := printf "%s-admin-credentials" .Release.Name }} +{{- /* The admin reads these into its environment when it starts and nothing + restarts it on a Secret change, so a password regenerated on every render + would leave the Secret holding one the running server does not accept. + Keep the one already issued. */}} +{{- $password := randAlphaNum 40 | b64enc }} +{{- with lookup "v1" "Secret" .Release.Namespace $name }} +{{- with (dig "data" "adminPassword" "" .) }} +{{- $password = . }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} +type: Opaque +data: + adminUser: {{ "admin" | b64enc }} + adminPassword: {{ $password }} +{{- end }} diff --git a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml index dfd408dad..eb038fe8f 100644 --- a/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml +++ b/packages/extra/seaweedfs/templates/dashboard-resourcemap.yaml @@ -27,6 +27,10 @@ rules: - {{ $.Release.Name }}-filer - {{ $.Release.Name }}-db - {{ $.Release.Name }}-s3 + {{- if .Values.lifecycle.enabled }} + - {{ $.Release.Name }}-admin + - {{ $.Release.Name }}-worker + {{- end }} {{- if eq .Values.topology "Simple" }} - {{ $.Release.Name }}-volume {{- range $poolName, $pool := .Values.volume.pools }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs-db.yaml b/packages/extra/seaweedfs/templates/seaweedfs-db.yaml index c86677c4d..261fcc93e 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs-db.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs-db.yaml @@ -42,6 +42,6 @@ spec: resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.db.resourcesPreset .Values.db.resources $) | nindent 8 }} size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 27e5557e0..92b40ef4d 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -214,10 +214,21 @@ spec: namespace: cozy-system interval: 30s timeout: 10m + {{- /* Flux remediates a failed install by uninstalling and a failed upgrade + by rolling back, and with retries: -1 it does so on every attempt. A + release waiting on tenant quota for the admin volume would then roll + the s3 pods back and forth with the [grpc.s3] restart annotation. + RetryOnFailure keeps what was applied and retries it. */}} install: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 upgrade: + strategy: + name: RetryOnFailure + retryInterval: 30s remediation: retries: -1 valuesFrom: @@ -249,7 +260,7 @@ spec: type: "persistentVolumeClaim" size: "{{ .Values.volume.size }}" {{- with .Values.volume.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 {{- if .Values.volume.diskType }} @@ -268,7 +279,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $pool.size | default $.Values.volume.size }}" {{- with ($pool.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 extraArgs: @@ -284,7 +295,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $zone.size | default $.Values.volume.size }}" {{- with ($zone.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 nodeSelector: | @@ -309,7 +320,7 @@ spec: type: "persistentVolumeClaim" size: "{{ $pool.size | default $zone.size | default $.Values.volume.size }}" {{- with ($pool.storageClass | default $zone.storageClass | default $.Values.volume.storageClass) }} - storageClass: "{{ . }}" + storageClass: {{ . | quote }} {{- end }} maxVolumes: 0 nodeSelector: | @@ -338,6 +349,7 @@ spec: s3: replicas: {{ .Values.s3.replicas }} ingress: + enabled: {{ not (empty $ingress) }} className: {{ $ingress }} host: {{ .Values.host | default (printf "s3.%s" $host) }} annotations: @@ -370,6 +382,16 @@ spec: secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} + admin: + enabled: {{ .Values.lifecycle.enabled }} + secret: + existingSecret: {{ .Release.Name }}-admin-credentials + {{- with .Values.lifecycle.storageClass }} + data: + storageClass: {{ . | quote }} + {{- end }} + worker: + enabled: {{ .Values.lifecycle.enabled }} cosi: driverName: "{{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io" bucketClassName: "{{ .Release.Namespace }}" @@ -502,4 +524,22 @@ spec: app.kubernetes.io/component: s3 app.kubernetes.io/name: seaweedfs version: {{ $.Chart.Version }} +{{- if .Values.lifecycle.enabled }} +{{- range $component := list "admin" "worker" }} +--- +apiVersion: cozystack.io/v1alpha1 +kind: WorkloadMonitor +metadata: + name: {{ $.Release.Name }}-{{ $component }} +spec: + replicas: 1 + minReplicas: 1 + kind: seaweedfs + type: {{ $component }} + selector: + app.kubernetes.io/component: {{ $component }} + app.kubernetes.io/name: seaweedfs + version: {{ $.Chart.Version }} +{{- end }} +{{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/tests/db_storage_class_test.yaml b/packages/extra/seaweedfs/tests/db_storage_class_test.yaml new file mode 100644 index 000000000..da4f6cd2d --- /dev/null +++ b/packages/extra/seaweedfs/tests/db_storage_class_test.yaml @@ -0,0 +1,30 @@ +suite: seaweedfs metadata database storage class reaches its release as a string + +templates: + - templates/seaweedfs-db.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + # A StorageClass name may be all digits; handed on unquoted it reaches the + # database release as a number, and a line break in it would open new keys + # in that release's values. + - it: hands a numeric storage class name on as a string + set: + db: + storageClass: "123" + asserts: + - equal: + path: spec.values.db.storageClass + value: "123" + + - it: keeps a storage class with a line break inside its own field + set: + db: + storageClass: "local\n replicas: 9" + asserts: + - equal: + path: spec.values.db.storageClass + value: "local\n replicas: 9" diff --git a/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml b/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml new file mode 100644 index 000000000..55acf8198 --- /dev/null +++ b/packages/extra/seaweedfs/tests/ingress_dependency_test.yaml @@ -0,0 +1,90 @@ +suite: seaweedfs waits for ingress-nginx only in a namespace that runs it +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: + topology: Simple + replicationFactor: 2 + _cluster: + issuer-name: letsencrypt-prod + +tests: + - it: depends on ingress-nginx-system when the namespace runs its own ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: tenant-root + asserts: + - contains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-root + + - it: does not depend on ingress-nginx-system when the root tenant has no ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: "" + asserts: + - notContains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-root + + - it: does not depend on ingress-nginx-system when ingress lives in a parent namespace + release: + namespace: tenant-acme + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: acme.example.org + ingress: tenant-root + asserts: + - notContains: + path: spec.dependsOn + content: + name: ingress-nginx-system + namespace: tenant-acme + + - it: disables the S3 Ingress when the namespace has no ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: "" + asserts: + - equal: + path: spec.values.seaweedfs.s3.ingress.enabled + value: false + + - it: keeps the S3 Ingress when the namespace has an ingress + documentSelector: + path: metadata.name + value: seaweedfs-system + set: + _namespace: + host: example.org + ingress: tenant-root + asserts: + - equal: + path: spec.values.seaweedfs.s3.ingress.enabled + value: true + - equal: + path: spec.values.seaweedfs.s3.ingress.className + value: tenant-root diff --git a/packages/extra/seaweedfs/tests/lifecycle_test.yaml b/packages/extra/seaweedfs/tests/lifecycle_test.yaml new file mode 100644 index 000000000..633514280 --- /dev/null +++ b/packages/extra/seaweedfs/tests/lifecycle_test.yaml @@ -0,0 +1,330 @@ +suite: seaweedfs lifecycle switch reaches the data plane + +# lifecycle.enabled decides whether the data-plane release runs the admin +# server and the worker that enforce bucket lifecycle rules. Both flags are +# sent either way, so the data-plane chart's own default never decides. + +templates: + - templates/seaweedfs.yaml + - templates/admin-credentials.yaml + - templates/dashboard-resourcemap.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + # The upstream walker deletes without an object identity, so a key + # overwritten between its listing and the delete RPC goes too. Applying + # stored rules is therefore an operator's decision, not an upgrade's. + - it: keeps the lifecycle admin and worker off by default + template: templates/seaweedfs.yaml + set: &platform + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: false + - equal: + path: spec.values.seaweedfs.worker.enabled + value: false + + - it: runs the lifecycle admin and worker when lifecycle is enabled + template: templates/seaweedfs.yaml + set: &enabled + <<: *platform + lifecycle: + enabled: true + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: true + - equal: + path: spec.values.seaweedfs.worker.enabled + value: true + - equal: + path: spec.values.seaweedfs.admin.secret.existingSecret + value: seaweedfs-admin-credentials + + - it: turns both off when lifecycle is disabled + template: templates/seaweedfs.yaml + set: + <<: *platform + lifecycle: + enabled: false + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.enabled + value: false + - equal: + path: spec.values.seaweedfs.worker.enabled + value: false + + # volume.storageClass is chosen for volume data, and in MultiZone it can be + # a class local to one zone, while the admin is not pinned to any zone. Its + # volume therefore takes its own class, and the cluster default otherwise. + - it: puts the admin volume on the lifecycle storage class + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: replicated + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.data.storageClass + value: replicated + + - it: does not put the admin volume on the volume storage class + template: templates/seaweedfs.yaml + set: + <<: *enabled + topology: MultiZone + replicationFactor: 1 + volume: + storageClass: zone-a-local + zones: + a: {} + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - notExists: + path: spec.values.seaweedfs.admin.data + + - it: leaves the admin volume on the cluster default when lifecycle names none + template: templates/seaweedfs.yaml + set: *enabled + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - notExists: + path: spec.values.seaweedfs.admin.data + + # A StorageClass name may be all digits. Handed on unquoted it reaches the + # data-plane release as a number, and the claim it ends up in is rejected. + - it: hands a numeric storage class name on as a string + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "123" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.admin.data.storageClass + value: "123" + + - it: refuses a storage class that is not a StorageClass name + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "replicated\n image:\n repository: example/other" + asserts: + - failedTemplate: + errorPattern: "/lifecycle/storageClass" + + - it: refuses a storage class longer than a StorageClass name can be + template: templates/seaweedfs.yaml + set: + <<: *enabled + lifecycle: + enabled: true + storageClass: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - failedTemplate: + errorPattern: "/lifecycle/storageClass" + + - it: monitors the admin and the worker while lifecycle is on + template: templates/seaweedfs.yaml + set: *enabled + asserts: + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-admin + any: true + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-worker + any: true + - equal: + path: spec.selector + value: + app.kubernetes.io/component: worker + app.kubernetes.io/name: seaweedfs + documentSelector: + path: metadata.name + value: seaweedfs-worker + - equal: + path: spec.selector + value: + app.kubernetes.io/component: admin + app.kubernetes.io/name: seaweedfs + documentSelector: + path: metadata.name + value: seaweedfs-admin + + - it: does not monitor them when lifecycle is off + template: templates/seaweedfs.yaml + set: + <<: *platform + lifecycle: + enabled: false + asserts: + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-admin + any: true + not: true + - containsDocument: + kind: WorkloadMonitor + apiVersion: cozystack.io/v1alpha1 + name: seaweedfs-worker + any: true + not: true + + - it: generates admin credentials for a new instance + template: templates/admin-credentials.yaml + set: *enabled + asserts: + - isKind: + of: Secret + - equal: + path: metadata.name + value: seaweedfs-admin-credentials + - equal: + path: data.adminUser + value: YWRtaW4= + - matchRegex: + path: data.adminPassword + pattern: '^.{40,}$' + + # A new password on every render would leave the running admin on the old + # one, since it reads the Secret into its environment once at start. + - it: keeps the password of an existing Secret + template: templates/admin-credentials.yaml + set: *enabled + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: seaweedfs-admin-credentials + namespace: tenant-root + data: + adminUser: YWRtaW4= + adminPassword: a2VwdA== + asserts: + - equal: + path: data.adminPassword + value: a2VwdA== + + - it: renders no credentials when lifecycle is off + template: templates/admin-credentials.yaml + set: + <<: *platform + lifecycle: + enabled: false + asserts: + - hasDocuments: + count: 0 + + # A Client instance runs no data plane of its own; its rules are enforced + # where the master it points at runs. + - it: renders no credentials for a Client instance + template: templates/admin-credentials.yaml + set: + <<: *enabled + topology: Client + filer: + grpcHost: seaweedfs-filer.tenant-other.svc + asserts: + - hasDocuments: + count: 0 + + # The admin PVC and the two lifecycle pods can leave an upgrade waiting on a + # full tenant quota. Under Flux's default rollback remediation each failed + # attempt would roll back the [grpc.s3] section and its restart annotation + # and the next one would bring them back, restarting the s3 pods on every + # cycle. Retrying in place keeps what was applied while the release waits. + - it: retries the data-plane release in place instead of rolling it back + template: templates/seaweedfs.yaml + set: *platform + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.install.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.strategy.name + value: RetryOnFailure + - equal: + path: spec.upgrade.remediation.retries + value: -1 + + # The dashboard reads each WorkloadMonitor by name through this Role, so a + # monitor missing from it never shows up for the tenant. + - it: lets the dashboard read the admin and worker monitors + template: templates/dashboard-resourcemap.yaml + set: *enabled + documentSelector: + path: kind + value: Role + asserts: + - contains: + path: rules[2].resourceNames + content: seaweedfs-admin + - contains: + path: rules[2].resourceNames + content: seaweedfs-worker + + - it: grants no lifecycle monitors when lifecycle is off + template: templates/dashboard-resourcemap.yaml + set: + <<: *platform + lifecycle: + enabled: false + documentSelector: + path: kind + value: Role + asserts: + - notContains: + path: rules[2].resourceNames + content: seaweedfs-admin + - notContains: + path: rules[2].resourceNames + content: seaweedfs-worker diff --git a/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml b/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml new file mode 100644 index 000000000..2474f3358 --- /dev/null +++ b/packages/extra/seaweedfs/tests/volume_storage_class_test.yaml @@ -0,0 +1,102 @@ +suite: seaweedfs volume storage class reaches the data plane as a string + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: &platform + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: hands a numeric storage class name on as a string + set: + <<: *platform + volume: + storageClass: "123" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volume.dataDirs[0].storageClass + value: "123" + + # The value is written into the values of another release; a line break in + # it must not open new keys there. + - it: keeps a storage class with a line break inside its own field + set: + <<: *platform + volume: + storageClass: "local\n image: example/other" + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volume.dataDirs[0].storageClass + value: "local\n image: example/other" + - notExists: + path: spec.values.seaweedfs.volume.dataDirs[0].image + + # A pool or zone class is quoted by hand around the value, which a quote + # character inside it closes early, breaking the release values. + - it: keeps a pool storage class with a quote inside its own field + set: + <<: *platform + volume: + pools: + fast: + diskType: ssd + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.fast.dataDirs[0].storageClass + value: 'local"x' + + - it: keeps a zone storage class with a quote inside its own field + set: + <<: *platform + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.dataDirs[0].storageClass + value: 'local"x' + + - it: keeps a zone pool storage class with a quote inside its own field + set: + <<: *platform + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + pools: + fast: + diskType: ssd + storageClass: 'local"x' + documentSelector: + path: metadata.name + value: seaweedfs-system + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a-fast.dataDirs[0].storageClass + value: 'local"x' diff --git a/packages/extra/seaweedfs/values.schema.json b/packages/extra/seaweedfs/values.schema.json index ab9b953ac..0df1bef8f 100644 --- a/packages/extra/seaweedfs/values.schema.json +++ b/packages/extra/seaweedfs/values.schema.json @@ -868,6 +868,25 @@ ] } } + }, + "lifecycle": { + "description": "S3 bucket lifecycle configuration.", + "type": "object", + "default": {}, + "properties": { + "enabled": { + "description": "Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new.", + "type": "boolean", + "default": false + }, + "storageClass": { + "description": "StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again.", + "type": "string", + "default": "", + "maxLength": 253, + "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$" + } + } } } } diff --git a/packages/extra/seaweedfs/values.yaml b/packages/extra/seaweedfs/values.yaml index 0aa569bc4..fdc58a11e 100644 --- a/packages/extra/seaweedfs/values.yaml +++ b/packages/extra/seaweedfs/values.yaml @@ -192,3 +192,21 @@ s3: replicas: 2 resources: {} resourcesPreset: "t1.small" + +## @typedef {struct} Lifecycle - Enforcement of S3 bucket lifecycle rules. +## @field {bool} [enabled] - Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new. +## @field {string} [storageClass] - StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again. +## @maxLength 253 +## @pattern ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$ + +## @param {Lifecycle} [lifecycle] - S3 bucket lifecycle configuration. +lifecycle: + # Off by default. The daily walk of SeaweedFS (4.45 through 4.52) sends its + # expiry deletes without an object identity, which the s3 server takes as + # "skip the check" (weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go, + # identityMatches in weed/s3api/s3api_internal_lifecycle.go), and it does not + # re-check the object's age. A key overwritten between the walk's listing and + # the delete loses its new content, which a tenant bucket that rewrites a + # fixed key can hit. On by default only once that path carries identity. + enabled: false + storageClass: "" diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index 6665cc639..b481d1e1c 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -101,8 +101,11 @@ https://. - provisionBucket: true + Secret missing (offline `helm template`/unit render, or pre-reconcile install where lookup returns nil) → fall back - to .Values.backupStorage.endpoint. On a live deploy Flux re-renders on - spec.interval once the Secret exists, promoting the derived endpoint. + to .Values.backupStorage.endpoint. Flux does not re-render on + spec.interval; the controller's DefaultObjectsGate forces the render + once the Secret exists, and again whenever the Secret's endpoint stops + matching the backups.cozystack.io/endpoint annotation every default + Strategy CR and the Velero BSL carry with this helper's output. Normalization of the decoded value: trim first (a trailing newline in the Secret data would otherwise ride into the URL and break it), then strip any @@ -187,3 +190,28 @@ true {{- end -}} {{- end -}} {{- end -}} + +{{/* + backupStorage.forcePathStyle as "true", "false", or "" when unset. Its + consumers read it differently: the controller and Velero parse it with + strconv.ParseBool, the etcd and MongoDB strategies write it into a boolean + field that Helm decodes as YAML 1.1 (so "yes" or "off" is a boolean there), + and the Kafka and RabbitMQ scripts compare the string with "false". + Normalising it here means a spelling cannot mean path style to some drivers + and not to others. The spellings accepted are those of strconv.ParseBool and + of YAML 1.1 booleans, a superset of the boolean spellings the etcd and + MongoDB strategies accepted; any other value fails the render. +*/}} +{{- define "backupstrategy-controller.forcePathStyle" -}} +{{- $value := .Values.backupStorage.forcePathStyle -}} +{{- if not (kindIs "invalid" $value) -}} +{{- $raw := toString $value -}} +{{- if has $raw (list "1" "t" "T" "true" "True" "TRUE" "y" "Y" "yes" "Yes" "YES" "on" "On" "ON") -}} +true +{{- else if has $raw (list "0" "f" "F" "false" "False" "FALSE" "n" "N" "no" "No" "NO" "off" "Off" "OFF") -}} +false +{{- else if ne $raw "" -}} +{{- fail (printf "backupStorage.forcePathStyle must be a boolean or unset, got %q" $raw) -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index f35e97661..2fc9b9ee6 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -47,7 +47,7 @@ spec: - name: BACKUP_STORAGE_REGION value: {{ .Values.backupStorage.region | quote }} - name: BACKUP_STORAGE_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} - name: BACKUP_STORAGE_SYSTEM_NAMESPACES value: {{ .Values.backupStorage.systemNamespaces | join "," | quote }} {{- if .Values.backupStorage.reconcileDefaultObjects }} @@ -87,6 +87,18 @@ spec: value: {{ printf "bucket-%s-system" .Values.backupStorage.bucketName | quote }} - name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE value: {{ .Values.backupStorage.namespace | quote }} + # The COSI Secret that release renders the credentials Secret from + # (packages/system/bucket/templates/user-credentials.yaml appends + # "-credentials" to its name). The COSI sidecar writes it once per + # BucketAccess, so the gate compares the two endpoints and forces the + # release when the credentials Secret lags behind. Only the system + # user's Secret of this very bucket is rendered by that release; any + # other systemSecretName would be forced without ever changing. + {{- $cosiSecretName := printf "bucket-%s-system" .Values.backupStorage.bucketName }} + {{- if eq .Values.backupStorage.systemSecretName (printf "%s-credentials" $cosiSecretName) }} + - name: BACKUP_CREDENTIALS_COSI_SECRET_NAME + value: {{ $cosiSecretName | quote }} + {{- end }} {{- end }} {{- if .Values.velero.bslEnabled }} - name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 331a89be8..3e6747966 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -18,6 +18,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Altinity metadata: name: cozy-default-altinity + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: spec: diff --git a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml index 58dda61b1..bf34b2e0d 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml @@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: CNPG metadata: name: cozy-default-cnpg + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: {{- /* diff --git a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml index f335f04cf..e7b2afcad 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml @@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Etcd metadata: name: cozy-default-etcd + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: destination: @@ -13,7 +15,7 @@ spec: endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} key: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}/" | quote }} region: {{ .Values.backupStorage.region | quote }} - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} # The etcd-operator reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY # from this Secret by default (no explicit key fields like the # MariaDB strategy's *SecretKeyRef). The projector writes exactly diff --git a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml index 061be2ffb..82f9ab6ca 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml @@ -8,6 +8,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: FoundationDB metadata: name: cozy-default-foundationdb + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: blobStoreConfiguration: diff --git a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml index c65462295..fb015ba66 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml @@ -23,6 +23,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Kafka metadata: name: cozy-default-kafka + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: artifactURITemplate: {{ printf "s3://%s/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/kafka-metadata.txt" $bucketName | quote }} template: @@ -67,7 +69,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/usr/bin/bash", "-c"] # The driver script is kept as a standalone shell file and # embedded verbatim so hack/kafka-backup-script.bats can execute diff --git a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml index ec8b35160..7615a8b0d 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml @@ -7,6 +7,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: MariaDB metadata: name: cozy-default-mariadb + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: storage: diff --git a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml index 7509f4bf7..3bff9472b 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml @@ -54,6 +54,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: MongoDB metadata: name: cozy-default-mongodb + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: storageName: s3-storage @@ -74,6 +76,6 @@ spec: prefix: {{ printf "{{ .Application.metadata.namespace }}/{{ .Application.metadata.name }}" | quote }} region: {{ .Values.backupStorage.region | quote }} credentialsSecret: cozy-backups-creds - forcePathStyle: {{ .Values.backupStorage.forcePathStyle }} + forcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . }} insecureSkipTLSVerify: {{ .Values.backupStorage.insecureSkipTLSVerify }} {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml index 9c303a6d3..28c9250f8 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml @@ -26,6 +26,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Rabbitmq metadata: name: cozy-default-rabbitmq + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: # The driver defers the run until the RabbitmqCluster reports AllReplicasReady # (a Go-side precondition), so a not-yet-ready broker surfaces a precise @@ -96,7 +98,7 @@ spec: - name: S3_REGION value: {{ .Values.backupStorage.region | quote }} - name: S3_FORCE_PATH_STYLE - value: {{ .Values.backupStorage.forcePathStyle | quote }} + value: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} command: ["/bin/sh", "-c"] args: - | diff --git a/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml index 3db27f7e2..dc930f394 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml @@ -21,6 +21,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Redis metadata: name: cozy-default-redis + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: spec: diff --git a/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml index 4511fbc58..1c1a7b9b5 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml @@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Velero metadata: name: cozy-default-velero-vmdisk + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: restoreSpec: diff --git a/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml index b1f919125..9666a7cce 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml @@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1 kind: Velero metadata: name: cozy-default-velero-vminstance + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: template: restoreSpec: diff --git a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml index e0ba05ad8..a01f480ac 100644 --- a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml +++ b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml @@ -12,6 +12,8 @@ kind: BackupStorageLocation metadata: name: cozy-default namespace: {{ .Values.velero.namespace }} + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} spec: provider: aws # default=false: do not silently displace an admin-provisioned default @@ -29,7 +31,7 @@ spec: key: cloud config: region: {{ .Values.backupStorage.region | quote }} - s3ForcePathStyle: {{ .Values.backupStorage.forcePathStyle | quote }} + s3ForcePathStyle: {{ include "backupstrategy-controller.forcePathStyle" . | quote }} s3Url: {{ include "backupstrategy-controller.endpoint" . | quote }} {{- end }} {{- end }} diff --git a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml index 976fee166..a222d66ee 100644 --- a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml +++ b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml @@ -11,6 +11,8 @@ suite: DefaultObjectsGate wiring — the release the gate forces, and its RBAC templates: - templates/deployment.yaml - templates/rbac.yaml + - templates/strategy-cnpg-default.yaml + - templates/velero-bsl.yaml release: name: backupstrategy-controller @@ -61,6 +63,16 @@ tests: content: name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE value: tenant-root + # The COSI Secret the bucket release renders the credentials Secret + # from is the BucketAccess credentialsSecretName, -system, + # which is also BACKUP_STORAGE_SECRET_NAME minus "-credentials" + # (packages/system/bucket/templates/user-credentials.yaml). The gate + # compares the two endpoints. + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_COSI_SECRET_NAME + value: bucket-cozy-backups-system # BACKUP_STORAGE_SECRET_NAME is exactly the release name above with # "-credentials" appended — that is what makes forcing this release the # thing that creates that Secret. @@ -86,6 +98,11 @@ tests: content: name: BACKUP_STORAGE_SECRET_NAME value: bucket-platform-backups-system-credentials + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_COSI_SECRET_NAME + value: bucket-platform-backups-system # External S3: the Secret is admin-managed and no release renders it, so # there is nothing to force. Handing the gate stale coordinates would make @@ -105,6 +122,39 @@ tests: content: name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE value: tenant-root + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_COSI_SECRET_NAME + value: bucket-cozy-backups-system + + # A systemSecretName that is not -credentials is not rendered by + # the bucket chart from a COSI Secret, so there is nothing to compare it to. + - it: "systemSecretName without the -credentials suffix: no COSI Secret to compare against" + template: templates/deployment.yaml + set: + backupStorage.systemSecretName: bucket-cozy-backups-system + asserts: + - notContains: + path: spec.template.spec.containers[0].env + any: true + content: + name: BACKUP_CREDENTIALS_COSI_SECRET_NAME + + # The forced release renders only -credentials for its own system + # user. A systemSecretName naming another bucket's Secret is not refreshed by + # forcing bucket--system, so comparing it would force that + # release every MinForceInterval and change nothing. + - it: "systemSecretName of another bucket: no COSI Secret to compare against" + template: templates/deployment.yaml + set: + backupStorage.systemSecretName: bucket-other-system-credentials + asserts: + - notContains: + path: spec.template.spec.containers[0].env + any: true + content: + name: BACKUP_CREDENTIALS_COSI_SECRET_NAME - it: "velero.bslEnabled=false: the BSL is not rendered, so the gate must not look for it" template: templates/deployment.yaml @@ -154,3 +204,41 @@ tests: apiGroups: ["velero.io"] resources: ["backupstoragelocations"] verbs: ["get"] + + # The gate compares backups.cozystack.io/endpoint with the endpoint it + # derives from the credentials Secret (renderedEndpointFor in + # internal/backupcontroller/default_objects_gate.go, whose test uses the + # same input). Any divergence would make every object look stale and force + # this release every MinForceInterval. + - it: "objects carry the endpoint derived from the credentials Secret as their endpoint annotation" + set: + backupStorage.bucketNameOverride: test-bucket + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: bucket-cozy-backups-system-credentials + namespace: tenant-root + data: + # "http://s3.example.com:8333\n" + endpoint: aHR0cDovL3MzLmV4YW1wbGUuY29tOjgzMzMK + asserts: + - equal: + path: metadata.annotations["backups.cozystack.io/endpoint"] + value: https://s3.example.com:8333 + template: templates/strategy-cnpg-default.yaml + - equal: + path: spec.template.barmanObjectStore.endpointURL + value: https://s3.example.com:8333 + template: templates/strategy-cnpg-default.yaml + - equal: + path: metadata.annotations["backups.cozystack.io/endpoint"] + value: https://s3.example.com:8333 + template: templates/velero-bsl.yaml diff --git a/packages/system/backupstrategy-controller/tests/deployment_test.yaml b/packages/system/backupstrategy-controller/tests/deployment_test.yaml index 6c17f5f58..f92455a8d 100644 --- a/packages/system/backupstrategy-controller/tests/deployment_test.yaml +++ b/packages/system/backupstrategy-controller/tests/deployment_test.yaml @@ -54,3 +54,12 @@ tests: name: BACKUP_STORAGE_SECRET_NAMESPACE value: tenant-root any: true + + # A value no consumer took as a boolean is refused once, at render time; see + # tests/force_path_style_test.yaml for what each consumer receives. + - it: "forcePathStyle that is no boolean spelling fails the render and names the value" + set: + backupStorage.forcePathStyle: "maybe" + asserts: + - failedTemplate: + errorPattern: 'backupStorage\.forcePathStyle must be a boolean or unset, got "maybe"' diff --git a/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml new file mode 100644 index 000000000..eda742951 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/force_path_style_test.yaml @@ -0,0 +1,501 @@ +suite: forcePathStyle — one normalised boolean for every consumer + +# backupStorage.forcePathStyle reaches drivers that read it differently: the +# controller and Velero parse it with strconv.ParseBool, the etcd and MongoDB +# strategies write it into a boolean field Helm decodes as YAML 1.1, and the +# Kafka and RabbitMQ scripts compare the string with "false". The chart accepts +# the spellings of both strconv.ParseBool and YAML 1.1, hands each consumer +# "true" or "false", and refuses anything else, so a spelling cannot mean path +# style to one driver and not to another. + +templates: + - templates/deployment.yaml + - templates/strategy-etcd-default.yaml + - templates/strategy-mongodb-default.yaml + - templates/strategy-kafka-default.yaml + - templates/strategy-rabbitmq-default.yaml + - templates/velero-bsl.yaml + +release: + name: t + namespace: cozy-backup-controller + +set: + backupStorage.bucketNameOverride: test-bucket + +tests: + - it: "forcePathStyle '1' reaches the controller as true" + set: + backupStorage.forcePathStyle: "1" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 't' reaches the controller as true" + set: + backupStorage.forcePathStyle: "t" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'T' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "T" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'true' reaches the controller as true" + set: + backupStorage.forcePathStyle: "true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'True' reaches the controller as true" + set: + backupStorage.forcePathStyle: "True" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'TRUE' reaches the controller as true" + set: + backupStorage.forcePathStyle: "TRUE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'Y' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Y" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'yes' reaches every consumer as true" + set: + backupStorage.forcePathStyle: "yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: true + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "true" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "true" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Yes' reaches the controller as true" + set: + backupStorage.forcePathStyle: "Yes" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'YES' reaches the controller as true" + set: + backupStorage.forcePathStyle: "YES" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'on' reaches the controller as true" + set: + backupStorage.forcePathStyle: "on" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'On' reaches the controller as true" + set: + backupStorage.forcePathStyle: "On" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle 'ON' reaches the controller as true" + set: + backupStorage.forcePathStyle: "ON" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + + - it: "forcePathStyle '0' reaches the controller as false" + set: + backupStorage.forcePathStyle: "0" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'f' reaches the controller as false" + set: + backupStorage.forcePathStyle: "f" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'F' reaches the controller as false" + set: + backupStorage.forcePathStyle: "F" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'false' reaches the controller as false" + set: + backupStorage.forcePathStyle: "false" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'False' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "False" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'FALSE' reaches the controller as false" + set: + backupStorage.forcePathStyle: "FALSE" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'n' reaches the controller as false" + set: + backupStorage.forcePathStyle: "n" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'N' reaches the controller as false" + set: + backupStorage.forcePathStyle: "N" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'no' reaches the controller as false" + set: + backupStorage.forcePathStyle: "no" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'No' reaches the controller as false" + set: + backupStorage.forcePathStyle: "No" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'NO' reaches the controller as false" + set: + backupStorage.forcePathStyle: "NO" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'off' reaches every consumer as false" + set: + backupStorage.forcePathStyle: "off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: false + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-rabbitmq-default.yaml + - equal: + path: spec.config.s3ForcePathStyle + value: "false" + template: templates/velero-bsl.yaml + + - it: "forcePathStyle 'Off' reaches the controller as false" + set: + backupStorage.forcePathStyle: "Off" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle 'OFF' reaches the controller as false" + set: + backupStorage.forcePathStyle: "OFF" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "false" + template: templates/deployment.yaml + + - it: "forcePathStyle as a YAML boolean (the default) reaches every consumer as true" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "true" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + - it: "forcePathStyle as a YAML boolean false reaches the scripts as false" + set: + backupStorage.forcePathStyle: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_FORCE_PATH_STYLE + value: "false" + template: templates/strategy-kafka-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: false + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle as the number 1 reaches the boolean fields as true" + set: + backupStorage.forcePathStyle: 1 + asserts: + - equal: + path: spec.template.destination.s3.forcePathStyle + value: true + template: templates/strategy-etcd-default.yaml + + # Unset leaves each driver its own default, as before. + - it: "forcePathStyle null stays unset for every consumer" + set: + backupStorage.forcePathStyle: null + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml + - equal: + path: spec.template.s3.forcePathStyle + value: null + template: templates/strategy-mongodb-default.yaml + + - it: "forcePathStyle empty stays unset for every consumer" + set: + backupStorage.forcePathStyle: "" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_FORCE_PATH_STYLE + value: "" + template: templates/deployment.yaml + - equal: + path: spec.template.destination.s3.forcePathStyle + value: null + template: templates/strategy-etcd-default.yaml diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 6cf39c65a..39f45c741 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -105,10 +105,6 @@ backupStorage: endpoint: http://seaweedfs-s3.tenant-root.svc.cozy.local:8333 # region is re-projected into cozy-backups-creds on the next reconcile # (1-minute tick for system namespaces, on-demand for tenant namespaces). - # However, charts that consume the region via secretKeyRef in env vars — - # the chart-emitted ClickHouse sidecar on `useSystemBucket: true` is the - # only such consumer today — only pick up the new value on Pod restart. - # Plan a rollout of those Pods when you change this value. region: us-east-1 forcePathStyle: true # systemSecretName — the human-friendly Secret produced by the Bucket @@ -162,10 +158,19 @@ backupStorage: # credentials Secret is missing it forces the bucket's -system # release; once the bucket name resolves it checks that every object # cozy-default routes to exists and forces THIS release when any is - # missing. Both are throttled, independently, and a no-op in the steady - # state. A suspended release is skipped rather than re-stamped forever. The - # gate does NOT create the objects itself: their bodies are values-driven - # and Helm stays their single owner. + # missing. It repairs a stale S3 endpoint the same way: when the + # credentials Secret's endpoint differs from the COSI Secret it is rendered + # from it forces the bucket's -system release (only while + # systemSecretName is bucket--system-credentials, the one + # Secret that release renders), and when the + # backups.cozystack.io/endpoint annotation on the Velero BSL or a default + # Strategy differs from the credentials Secret's endpoint, or is absent, it + # forces THIS release. The COSI Secret itself keeps the endpoint it was granted with, so + # after a publishing.host change that endpoint follows only once the + # BucketAccess is re-granted. Both are throttled, independently, and a no-op + # in the steady state. A suspended release is skipped rather than + # re-stamped forever. The gate does NOT create the objects itself: their + # bodies are values-driven and Helm stays their single owner. # # Turn this off only if you manage the default Strategy CRs out of band and # accept that a missed install-time render is never repaired. Watch diff --git a/packages/system/bucket/templates/user-credentials.yaml b/packages/system/bucket/templates/user-credentials.yaml index 6c2a14bd1..a45320254 100644 --- a/packages/system/bucket/templates/user-credentials.yaml +++ b/packages/system/bucket/templates/user-credentials.yaml @@ -27,7 +27,9 @@ So the repair lives outside the chart, where it is per-object and costs a throttled annotation patch: backupstrategy-controller's DefaultObjectsGate (internal/backupcontroller/default_objects_gate.go) forces this release - when the platform bucket's credentials Secret is absent. That covers the + when the platform bucket's credentials Secret is absent, or when its + endpoint differs from the COSI Secret's (a BucketAccess re-granted after a + publishing.host change). That covers the cluster-breaking case — the cozy-backups Secret feeds the credentials projector, hence every default BackupClass strategy, Velero, and migration 50. Tenant buckets keep the pre-existing behaviour; generalising the gate diff --git a/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml b/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml index 560bc4598..72f24e1f9 100644 --- a/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml +++ b/packages/system/cozystack-basics/templates/cozystack-values-secret.yaml @@ -15,7 +15,13 @@ stringData: _namespace: host: {{ index .Values._cluster "root-host" | quote }} etcd: tenant-root + {{- /* Without ingress on the root Tenant there is no + ingress-nginx-system in tenant-root, so the key stays empty, as + it does for a sub-tenant with no ingress anywhere above it. A + missing key keeps tenant-root for Secrets written before it. */ -}} + {{- if eq (toString (index .Values._cluster "ingress-enabled" | default "true")) "true" }} ingress: tenant-root + {{- end }} {{- /* `gateway` is conditional, the other four `_namespace` keys are unconditional. The `apps/tenant` chart gates tenant-root via `ne (include "tenant.name" .) "tenant-root"`, so the namespace.yaml diff --git a/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml b/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml index 28a88b0e1..33db1dcd5 100644 --- a/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml +++ b/packages/system/cozystack-basics/tests/cozystack-values-secret_test.yaml @@ -19,14 +19,16 @@ set: # never reaches tenant-root — this Secret is the only place that # value gets set for the root tenant. # -# Four of the five `_namespace.*` keys (`etcd`, `ingress`, -# `monitoring`, `seaweedfs`) are always rendered, because their -# dependent platform apps are always installable in tenant-root. -# `gateway` is conditional on `_cluster.gateway-enabled`: when the -# platform-level flag is false (the default), Gateway is not -# installed, and writing `gateway: tenant-root` would point apps at a -# Gateway that doesn't exist. The fixtures below pin both halves of -# that contract. +# Three of the five `_namespace.*` keys (`etcd`, `monitoring`, +# `seaweedfs`) are always rendered, because their dependent platform +# apps are always installable in tenant-root. `gateway` is conditional +# on `_cluster.gateway-enabled`: when the platform-level flag is false +# (the default), Gateway is not installed, and writing +# `gateway: tenant-root` would point apps at a Gateway that doesn't +# exist. `ingress` is conditional on `_cluster.ingress-enabled` for the +# same reason: with ingress off on the root Tenant there is no +# ingress-nginx-system in tenant-root, and seaweedfs would wait on it +# forever. The fixtures below pin both halves of each contract. tests: - it: omits _namespace.gateway when _cluster.gateway-enabled is missing (default) asserts: @@ -52,6 +54,54 @@ tests: path: stringData["values.yaml"] pattern: '(?m)^\s+gateway: tenant-root$' + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is missing (Secrets written before the key existed) + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is "true" + set: + _cluster: + ingress-enabled: "true" + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: emits _namespace.ingress=tenant-root when _cluster.ingress-enabled is a bool true + set: + _cluster: + ingress-enabled: true + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress: tenant-root$' + + - it: omits _namespace.ingress when _cluster.ingress-enabled is "false" + set: + _cluster: + ingress-enabled: "false" + asserts: + - notMatchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+ingress:' + + - it: keeps _namespace.{etcd,monitoring,seaweedfs} when ingress is off + set: + _cluster: + ingress-enabled: "false" + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+etcd: tenant-root$' + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+monitoring: tenant-root$' + - matchRegex: + path: stringData["values.yaml"] + pattern: '(?m)^\s+seaweedfs: tenant-root$' + - it: keeps _namespace.{etcd,ingress,monitoring,seaweedfs} unconditional when Gateway is off — regression guard for a refactor that accidentally gates them on the same flag asserts: - matchRegex: diff --git a/packages/system/dashboard/images/console/Containerfile b/packages/system/dashboard/images/console/Containerfile index 84d634f46..d7493c60c 100644 --- a/packages/system/dashboard/images/console/Containerfile +++ b/packages/system/dashboard/images/console/Containerfile @@ -32,7 +32,14 @@ FROM nginxinc/nginx-unprivileged:1.29-alpine@sha256:0c79d56aee561a1d81c63f00eee5 COPY --from=builder /src/apps/console/dist /usr/share/nginx/html -COPY <<'EOF' /etc/nginx/conf.d/default.conf +# The config is a template the image's entrypoint renders into conf.d at +# startup, substituting only variables set in the environment: CLUSTER_DOMAIN, +# which the chart overrides with the platform cluster domain, and +# NGINX_LOCAL_RESOLVERS, which the entrypoint reads from the pod's resolv.conf. +ENV CLUSTER_DOMAIN=cozy.local +ENV NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1 + +COPY <<'EOF' /etc/nginx/templates/default.conf.template map $http_upgrade $connection_upgrade { default upgrade; '' close; @@ -54,6 +61,35 @@ server { root /usr/share/nginx/html; index index.html; + # Logs and metrics roots of a namespace or of one application, served by + # the observability proxy rather than the API server. A regex location wins + # over the /apis prefix below. The upstream is a variable so nginx resolves + # it per request: on a cluster without the proxy nginx still starts, and a + # request it cannot deliver gets the not-installed answer below. The + # resolver does not apply search domains, hence the fully qualified name. + location ~ ^/apis/(?:cozystack\.io/v1alpha1/namespaces/[^/]+/(?:logs|metrics)|apps\.cozystack\.io/v1alpha1/namespaces/[^/]+/[^/]+/[^/]+/(?:log|metrics))/ { + resolver ${NGINX_LOCAL_RESOLVERS}; + set $observability_proxy observability-proxy.cozy-observability-proxy.svc.${CLUSTER_DOMAIN}:80; + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + # With proxy_intercept_errors off this catches only the 502 and 504 + # nginx makes itself when the proxy cannot be resolved or reached; + # the proxy's own 502 and 504 for an unreachable store pass through. + proxy_intercept_errors off; + error_page 502 504 = @observability_proxy_unavailable; + proxy_pass http://$observability_proxy; + } + + # The console reads a Kubernetes Status body as "not installed", the same + # answer kube-apiserver gives for a root nothing serves. + location @observability_proxy_unavailable { + default_type application/json; + return 404 '{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the observability proxy is not available on this cluster","reason":"NotFound","code":404}'; + } + # Kubernetes API proxy for both core and aggregated groups. # Chunked-encoding watches require proxy_buffering off and a long timeout. location /apis { diff --git a/packages/system/dashboard/templates/console.yaml b/packages/system/dashboard/templates/console.yaml index c4e641578..c05f3ee4b 100644 --- a/packages/system/dashboard/templates/console.yaml +++ b/packages/system/dashboard/templates/console.yaml @@ -40,6 +40,11 @@ spec: containers: - image: {{ .Values.console.image | quote }} imagePullPolicy: IfNotPresent + {{- with index .Values._cluster "cluster-domain" }} + env: + - name: CLUSTER_DOMAIN + value: {{ . | quote }} + {{- end }} livenessProbe: failureThreshold: 3 httpGet: diff --git a/packages/system/dashboard/tests/console_cluster_domain_test.yaml b/packages/system/dashboard/tests/console_cluster_domain_test.yaml new file mode 100644 index 000000000..c5e605638 --- /dev/null +++ b/packages/system/dashboard/tests/console_cluster_domain_test.yaml @@ -0,0 +1,42 @@ +suite: console cluster domain +# The console nginx names the observability proxy by its fully qualified +# Service name, because the resolver it uses for a per-request upstream does +# not apply the pod's search domains. The image defaults to cozy.local; the +# chart passes the platform cluster domain so a cluster on another domain +# still reaches the proxy. +templates: + - templates/console.yaml + - templates/configmap.yaml + +release: + name: dashboard + namespace: cozy-dashboard + +tests: + - it: passes the platform cluster domain to the console + template: templates/console.yaml + set: + _cluster: + cluster-domain: cluster.local + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLUSTER_DOMAIN + value: cluster.local + + - it: leaves the image default when the platform passes no cluster domain + template: templates/console.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # The entrypoint renders the nginx template into /etc/nginx/conf.d at + # startup. On a read-only root it logs the failure and starts nginx on the + # stock default.conf, which serves no /apis, /api or proxy route at all. + - it: keeps the root filesystem writable for the rendered nginx config + template: templates/console.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: false diff --git a/packages/system/mariadb-operator/Makefile b/packages/system/mariadb-operator/Makefile index 905653cac..e80038778 100644 --- a/packages/system/mariadb-operator/Makefile +++ b/packages/system/mariadb-operator/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator diff --git a/packages/system/mariadb-operator/tests/cluster_name_test.yaml b/packages/system/mariadb-operator/tests/cluster_name_test.yaml new file mode 100644 index 000000000..45d762727 --- /dev/null +++ b/packages/system/mariadb-operator/tests/cluster_name_test.yaml @@ -0,0 +1,43 @@ +suite: mariadb-operator package resolves its pods in the platform cluster domain +release: + name: mariadb-operator + namespace: cozy-mariadb-operator +# core/platform hands the cluster domain to this package as +# mariadb-operator.clusterName in the shared values Secret. Guard that the +# vendored chart still threads that key into what the operator and the webhook +# certificate use, so a re-vendor that renames it goes red here. +tests: + - it: sets the operator CLUSTER_NAME from the injected domain + templates: + - charts/mariadb-operator/templates/operator/deployment.yaml + set: + mariadb-operator.clusterName: example.test + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLUSTER_NAME + value: example.test + + - it: issues the webhook certificate for the injected domain + templates: + - charts/mariadb-operator/templates/webhook/certificate.yaml + documentSelector: + path: kind + value: Certificate + set: + mariadb-operator.clusterName: example.test + asserts: + - contains: + path: spec.dnsNames + content: mariadb-operator-webhook.cozy-mariadb-operator.svc.example.test + + - it: keeps cozy.local as the standalone default + templates: + - charts/mariadb-operator/templates/operator/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLUSTER_NAME + value: cozy.local diff --git a/packages/system/mariadb-operator/values.yaml b/packages/system/mariadb-operator/values.yaml index 2728c084e..285975440 100644 --- a/packages/system/mariadb-operator/values.yaml +++ b/packages/system/mariadb-operator/values.yaml @@ -1,4 +1,6 @@ mariadb-operator: + # Fallback only: core/platform injects networking.clusterDomain into this key + # through the shared cozystack-values Secret (see its apps.yaml). clusterName: cozy.local metrics: enabled: true diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 00ab63c10..7a87fa5b5 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -10,7 +10,7 @@ spec: shardCount: 1 externalLabels: cluster: {{ .Values.vmagent.externalLabels.cluster }} - tenant: {{ .Values.global.target }} + tenant: {{ .Values.vmagent.externalLabels.tenant | default .Values.global.target | quote }} extraArgs: {{- toYaml (deepCopy .Values.vmagent.extraArgs | mergeOverwrite (fromYaml (include "monitoring-agents.vmagent.defaultExtraArgs" .))) | nindent 4 }} remoteWrite: diff --git a/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml b/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml new file mode 100644 index 000000000..894a1c053 --- /dev/null +++ b/packages/system/monitoring-agents/tests/vmagent_remote_write_test.yaml @@ -0,0 +1,65 @@ +suite: vmagent remote write targets and tenant label + +# The root bundle sets only global.target, and the chart defaults derive both +# the remote write URLs and the tenant label from it. A tenant Kubernetes +# cluster gets this chart through packages/apps/kubernetes instead, which +# passes vmagent.remoteWrite.urls and vmagent.externalLabels.tenant and leaves +# global.target at its default. If either value is dropped, vmagent writes to +# cozy-monitoring services that do not exist in the tenant cluster, or labels +# every series tenant="cozy-monitoring". +templates: + - templates/vmagent.yaml +release: + name: cozy-monitoring-agents + namespace: cozy-monitoring +tests: + - it: root install writes to both stores of global.target and labels series with it + set: + global.target: tenant-root + asserts: + - equal: + path: spec.remoteWrite + value: + - url: http://vminsert-shortterm.tenant-root.svc:8480/insert/0/prometheus + - url: http://vminsert-longterm.tenant-root.svc:8480/insert/0/prometheus + - equal: + path: spec.externalLabels.tenant + value: tenant-root + + - it: tenant cluster writes only to the URLs its parent app passes + set: + vmagent: + externalLabels: + cluster: kubernetes-clusterprod + tenant: tenant-a + remoteWrite: + urls: + - http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus + asserts: + - equal: + path: spec.remoteWrite + value: + - url: http://vminsert-shortterm.tenant-a.svc.cozy.local:8480/insert/0/prometheus + + - it: tenant cluster labels series with the owning tenant, not global.target + set: + vmagent: + externalLabels: + cluster: kubernetes-clusterprod + tenant: tenant-a + asserts: + - equal: + path: spec.externalLabels + value: + cluster: kubernetes-clusterprod + tenant: tenant-a + + - it: tenant label stays a string when the value reads as a YAML scalar + set: + vmagent: + externalLabels: + tenant: "true" + asserts: + - equal: + path: spec.externalLabels.tenant + value: "true" diff --git a/packages/system/monitoring/templates/grafana/grafana.yaml b/packages/system/monitoring/templates/grafana/grafana.yaml index 85517151a..2da6d2fc6 100644 --- a/packages/system/monitoring/templates/grafana/grafana.yaml +++ b/packages/system/monitoring/templates/grafana/grafana.yaml @@ -280,7 +280,7 @@ spec: path: auth.ini {{- end }} {{- end }} - {{- if ne $gatewayEnabled "true" }} + {{- if and (ne $gatewayEnabled "true") $ingress }} ingress: {{- /* Operator wildcard mode: nginx serves the default SSL cert, so skip per-host ACME. */}} {{- if not $wildcardSecret }} diff --git a/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml b/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml new file mode 100644 index 000000000..9e6b0350c --- /dev/null +++ b/packages/system/monitoring/tests/grafana_ingress_empty_class_test.yaml @@ -0,0 +1,40 @@ +suite: grafana ingress needs an ingress class to name +templates: + - templates/grafana/grafana.yaml + +release: + name: monitoring + namespace: tenant-root + +# An empty _namespace.ingress means no ingress-nginx serves this tenant, and +# the API rejects the ingressClassName "" the operator would put on the +# Ingress it creates from spec.ingress. + +tests: + - it: leaves spec.ingress out when the tenant has no ingress + set: + _cluster: + issuer-name: letsencrypt-prod + gateway-enabled: "false" + _namespace: + host: example.org + ingress: "" + asserts: + - equal: + path: kind + value: Grafana + - notExists: + path: spec.ingress + + - it: keeps spec.ingress when the tenant has an ingress + set: + _cluster: + issuer-name: letsencrypt-prod + gateway-enabled: "false" + _namespace: + host: example.org + ingress: tenant-root + asserts: + - equal: + path: spec.ingress.spec.ingressClassName + value: tenant-root diff --git a/packages/system/monitoring/tests/vtraces_collector_test.yaml b/packages/system/monitoring/tests/vtraces_collector_test.yaml index b450bf520..eaa5921cb 100644 --- a/packages/system/monitoring/tests/vtraces_collector_test.yaml +++ b/packages/system/monitoring/tests/vtraces_collector_test.yaml @@ -473,11 +473,18 @@ tests: path: spec.template.spec.containers[0].securityContext.capabilities.drop content: ALL + # The image name is not asserted: a release build swaps the upstream + # collector for the first-party otelcol-tracing one and restamps the tag. + # What must hold is that the value goes through cozy-lib.image (the cluster + # images-registry lands in front of it) and stays digest-pinned. - it: pins the collector image through the cozy-lib.image helper + set: + _cluster: + images-registry: mirror.example.com documentSelector: path: kind value: Deployment asserts: - matchRegex: path: spec.template.spec.containers[0].image - pattern: "opentelemetry-collector-contrib:.*@sha256:" + pattern: '^mirror\.example\.com/.+@sha256:[0-9a-f]{64}$' diff --git a/packages/system/observability-proxy/Chart.yaml b/packages/system/observability-proxy/Chart.yaml new file mode 100644 index 000000000..282c2234a --- /dev/null +++ b/packages/system/observability-proxy/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: cozy-observability-proxy +version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/observability-proxy/Makefile b/packages/system/observability-proxy/Makefile new file mode 100644 index 000000000..ee66c0656 --- /dev/null +++ b/packages/system/observability-proxy/Makefile @@ -0,0 +1,20 @@ +NAME=observability-proxy +NAMESPACE=cozy-observability-proxy + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +test: + helm unittest . + +image: image-observability-proxy + +image-observability-proxy: + docker buildx build -f images/observability-proxy/Dockerfile ../../.. \ + $(call image-tags,observability-proxy,v$(COZYSTACK_VERSION)) \ + $(call cache-args,observability-proxy) \ + --metadata-file images/observability-proxy.json \ + $(BUILDX_ARGS) + IMAGE="$(REGISTRY)/observability-proxy:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/observability-proxy.json -o json -r)" \ + yq -i '.observabilityProxy.image = strenv(IMAGE)' values.yaml + rm -f images/observability-proxy.json diff --git a/packages/system/observability-proxy/images/observability-proxy/Dockerfile b/packages/system/observability-proxy/images/observability-proxy/Dockerfile new file mode 100644 index 000000000..4615e74ec --- /dev/null +++ b/packages/system/observability-proxy/images/observability-proxy/Dockerfile @@ -0,0 +1,25 @@ +FROM --platform=$BUILDPLATFORM golang:1.27-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS builder +ARG GOPROXY + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace + +COPY go.mod go.sum ./ +RUN for attempt in 1 2 3; do GOOS=$TARGETOS GOARCH=$TARGETARCH go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done + +COPY api api/ +COPY pkg pkg/ +COPY cmd cmd/ +COPY internal internal/ + +RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -ldflags="-extldflags=-static" -o /observability-proxy ./cmd/observability-proxy + +FROM scratch + +COPY --from=builder /observability-proxy /observability-proxy +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +USER 65532:65532 +ENTRYPOINT ["/observability-proxy"] diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml new file mode 100644 index 000000000..581c15fde --- /dev/null +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -0,0 +1,68 @@ +{{- $clusterDomain := index (.Values._cluster | default dict) "cluster-domain" | default "cozy.local" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: observability-proxy + labels: + app.kubernetes.io/name: observability-proxy +spec: + replicas: {{ .Values.observabilityProxy.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: observability-proxy + template: + metadata: + labels: + app.kubernetes.io/name: observability-proxy + spec: + serviceAccountName: observability-proxy + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: observability-proxy + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: observability-proxy + image: {{ .Values.observabilityProxy.image | quote }} + args: + - --listen-address=:8080 + - --probe-address=:8081 + - --cluster-domain={{ $clusterDomain }} + - --root-tenant={{ .Values.observabilityProxy.rootTenant }} + - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: probes + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: probes + initialDelaySeconds: 15 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + {{- with .Values.observabilityProxy.resources }} + resources: {{- toYaml . | nindent 10 }} + {{- end }} diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml new file mode 100644 index 000000000..3c5392903 --- /dev/null +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy:observability-proxy +rules: +# Every request is authenticated and authorized against the API server. +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +# The namespace.cozystack.io/monitoring label names the logs store; it is +# read from an informer cache, which only lists and watches. +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cozy:observability-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:observability-proxy +subjects: +- kind: ServiceAccount + name: observability-proxy + namespace: {{ .Release.Namespace }} diff --git a/packages/system/observability-proxy/templates/sa.yaml b/packages/system/observability-proxy/templates/sa.yaml new file mode 100644 index 000000000..5a4eb2182 --- /dev/null +++ b/packages/system/observability-proxy/templates/sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: observability-proxy diff --git a/packages/system/observability-proxy/templates/service.yaml b/packages/system/observability-proxy/templates/service.yaml new file mode 100644 index 000000000..6a398532a --- /dev/null +++ b/packages/system/observability-proxy/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: observability-proxy + labels: + app.kubernetes.io/name: observability-proxy +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + app.kubernetes.io/name: observability-proxy diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml new file mode 100644 index 000000000..23460352f --- /dev/null +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -0,0 +1,14 @@ +# Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). +# No API server serves cozystack.io logs: the rule exists only for the +# SubjectAccessReview the observability proxy asks before it answers a read +# under /apis/cozystack.io/v1alpha1/namespaces//logs/. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cozy:observability:view + labels: + rbac.cozystack.io/aggregate-to-tenant-view: "true" +rules: +- apiGroups: ["cozystack.io"] + resources: ["logs"] + verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml new file mode 100644 index 000000000..ddf9d34b8 --- /dev/null +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -0,0 +1,109 @@ +suite: observability-proxy Deployment +# The flags, ports and pod label below are the contract the proxy binary, the +# Service and the console nginx are written against. A rename on either side +# breaks routing without failing a render. +templates: + - templates/deployment.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: labels the pods with the name the Service selects on + asserts: + - isKind: + of: Deployment + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: observability-proxy + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/name"] + value: observability-proxy + - equal: + path: spec.template.spec.serviceAccountName + value: observability-proxy + + - it: passes the listen, probe, root tenant and auth cache flags + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --listen-address=:8080 + - contains: + path: spec.template.spec.containers[0].args + content: --probe-address=:8081 + - contains: + path: spec.template.spec.containers[0].args + content: --root-tenant=tenant-root + - contains: + path: spec.template.spec.containers[0].args + content: --auth-cache-ttl=10s + + - it: falls back to cozy.local when the platform passes no cluster domain + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cozy.local + + - it: passes the platform cluster domain + set: + _cluster: + cluster-domain: cluster.local + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cluster.local + - notContains: + path: spec.template.spec.containers[0].args + content: --cluster-domain=cozy.local + + - it: exposes the API and probe ports + asserts: + - equal: + path: spec.template.spec.containers[0].ports + value: + - name: http + containerPort: 8080 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + + - it: probes /healthz and /readyz on the probe port + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet + value: + path: /healthz + port: probes + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet + value: + path: /readyz + port: probes + + - it: runs from the image in values + set: + observabilityProxy.image: registry.example/observability-proxy:dev@sha256:0000 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/observability-proxy:dev@sha256:0000 + + - it: satisfies the restricted pod security profile + asserts: + - equal: + path: spec.template.spec.securityContext + value: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + - equal: + path: spec.template.spec.containers[0].securityContext + value: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml new file mode 100644 index 000000000..af339b5d0 --- /dev/null +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -0,0 +1,67 @@ +suite: observability-proxy RBAC +# The proxy authenticates and authorizes every request through the API server, +# so a dropped verb here turns every request into an error at runtime rather +# than a render failure. The tenant role is what lets a namespace viewer pass +# the SubjectAccessReview the proxy asks for the namespace logs root. +templates: + - templates/rbac.yaml + - templates/tenant-clusterroles.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: lets the proxy review tokens and access, and list and watch namespaces + template: templates/rbac.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: rules + value: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] + + - it: binds the proxy role to the proxy ServiceAccount in the release namespace + template: templates/rbac.yaml + documentSelector: + path: kind + value: ClusterRoleBinding + asserts: + - equal: + path: roleRef + value: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cozy:observability-proxy + - equal: + path: subjects + value: + - kind: ServiceAccount + name: observability-proxy + namespace: cozy-observability-proxy + + - it: aggregates list on cozystack.io logs into the tenant view role + template: templates/tenant-clusterroles.yaml + asserts: + - isKind: + of: ClusterRole + - equal: + path: metadata.labels + value: + rbac.cozystack.io/aggregate-to-tenant-view: "true" + - equal: + path: rules + value: + - apiGroups: ["cozystack.io"] + resources: ["logs"] + verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/service_test.yaml b/packages/system/observability-proxy/tests/service_test.yaml new file mode 100644 index 000000000..123cc01ae --- /dev/null +++ b/packages/system/observability-proxy/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: observability-proxy Service +# The console nginx proxies to observability-proxy..svc on port 80, which +# must reach the pods' 8080, so the name, the port and the selector are pinned +# here. +templates: + - templates/service.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: serves port 80 named http on the pods' http port + asserts: + - isKind: + of: Service + - equal: + path: metadata.name + value: observability-proxy + - equal: + path: spec.ports + value: + - name: http + port: 80 + targetPort: http + protocol: TCP + - equal: + path: spec.selector + value: + app.kubernetes.io/name: observability-proxy diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml new file mode 100644 index 000000000..919e450a9 --- /dev/null +++ b/packages/system/observability-proxy/values.yaml @@ -0,0 +1,16 @@ +# Injected by the platform from the cozystack-values Secret. The empty +# default keeps the chart renderable on its own, outside a cluster. +_cluster: {} + +observabilityProxy: + image: "ghcr.io/cozystack/cozystack/observability-proxy:v0.0.0" + replicas: 2 + rootTenant: tenant-root + authCacheTTL: 10s + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 39adc0f58..0cd63edd2 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -30,7 +30,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml index a1d619c34..cfcaeb01d 100644 --- a/packages/system/redis-rd/cozyrds/redis.yaml +++ b/packages/system/redis-rd/cozyrds/redis.yaml @@ -8,7 +8,7 @@ spec: plural: redises singular: redis openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} release: prefix: redis- labels: diff --git a/packages/system/seaweedfs-db/templates/database.yaml b/packages/system/seaweedfs-db/templates/database.yaml index 807bd13af..136a3be83 100644 --- a/packages/system/seaweedfs-db/templates/database.yaml +++ b/packages/system/seaweedfs-db/templates/database.yaml @@ -11,7 +11,7 @@ spec: storage: size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} - storageClass: {{ . }} + storageClass: {{ . | quote }} {{- end }} monitoring: diff --git a/packages/system/seaweedfs-db/tests/database_test.yaml b/packages/system/seaweedfs-db/tests/database_test.yaml index 3eff5a112..2e6244cf1 100644 --- a/packages/system/seaweedfs-db/tests/database_test.yaml +++ b/packages/system/seaweedfs-db/tests/database_test.yaml @@ -60,3 +60,17 @@ tests: - equal: path: spec.inheritedMetadata.labels["policy.cozystack.io/allow-to-apiserver"] value: "true" + + # A StorageClass name may be all digits; emitted unquoted it renders as a + # number, which the Cluster rejects. + - it: keeps a numeric storage class name a string + release: + name: seaweedfs-db + namespace: cozy-seaweedfs + set: + db: + storageClass: "123" + asserts: + - equal: + path: spec.storage.storageClass + value: "123" diff --git a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml index 1fa5ad336..6cafe5c5d 100644 --- a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml +++ b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml @@ -8,7 +8,7 @@ spec: singular: seaweedfs plural: seaweedfses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"lifecycle":{"description":"S3 bucket lifecycle configuration.","type":"object","default":{},"properties":{"enabled":{"description":"Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. Rules stored before enabling are applied on the first pass, minutes after the switch. While on, in a bucket without versioning enabled, an object overwritten between the daily listing and its expiry delete can be deleted although it is new.","type":"boolean","default":false},"storageClass":{"description":"StorageClass of the admin server's 1Gi job-state volume. Empty uses the cluster default; `volume.storageClass` is not used, because in MultiZone it can be local to one zone. Prefer a replicated class: on a node-local one, losing the node stops lifecycle enforcement. Set it before enabling lifecycle: the class is part of the admin StatefulSet's volume claim template, which cannot change, so changing it while lifecycle is on makes every upgrade of the instance fail until it is reverted. To move the volume, disable lifecycle, delete the `admin-data-seaweedfs-admin-0` PVC of the instance, change the class and enable lifecycle again.","type":"string","default":"","maxLength":253,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$"}}}}} release: prefix: "" labels: @@ -26,7 +26,7 @@ spec: description: Seaweedfs module: true icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTEzOC42ODUgMTIxLjA1N0MxMzguNjg1IDEyNi42NTIgMTM2LjQ2MiAxMzIuMDE3IDEzMi41MDQgMTM1Ljk3M0MxMjguNTQ3IDEzOS45MjkgMTIzLjE3OSAxNDIuMTUxIDExNy41ODIgMTQyLjE1MUMxMTEuOTg1IDE0Mi4xNTEgMTA2LjYxOCAxMzkuOTI5IDEwMi42NiAxMzUuOTczQzk4LjcwMjggMTMyLjAxNyA5Ni40Nzk1IDEyNi42NTIgOTYuNDc5NSAxMjEuMDU3Qzk2LjQ3OTUgMTE4LjI4NyA5Ny4wMjUzIDExNS41NDQgOTguMDg1OCAxMTIuOTg1Qzk5LjE0NjMgMTEwLjQyNSAxMDAuNzAxIDEwOC4xIDEwMi42NiAxMDYuMTQxQzEwNC42MiAxMDQuMTgyIDEwNi45NDYgMTAyLjYyOSAxMDkuNTA3IDEwMS41NjlDMTEyLjA2NyAxMDAuNTA5IDExNC44MTEgOTkuOTYyOSAxMTcuNTgyIDk5Ljk2MjlDMTIwLjM1MyA5OS45NjI5IDEyMy4wOTggMTAwLjUwOSAxMjUuNjU4IDEwMS41NjlDMTI4LjIxOCAxMDIuNjI5IDEzMC41NDQgMTA0LjE4MiAxMzIuNTA0IDEwNi4xNDFDMTM0LjQ2NCAxMDguMSAxMzYuMDE4IDExMC40MjUgMTM3LjA3OSAxMTIuOTg1QzEzOC4xMzkgMTE1LjU0NCAxMzguNjg1IDExOC4yODcgMTM4LjY4NSAxMjEuMDU3WiIgZmlsbD0idXJsKCNwYWludDFfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMTEwLjMzNiAxMjYuMTQ3SDEyMy42OFYxMzYuODIzSDExMC4zMzZWMTI2LjE0N1oiIGZpbGw9IiMwRjVFOUMiLz4KPHBhdGggZD0iTTExOS4yNyAxMTIuMjk0QzExOS4yNyAxMTIuNzE0IDExOS4xNzggMTEzLjEzMSAxMTkgMTEzLjUxOUMxMTguODIxIDExMy45MDggMTE4LjU2IDExNC4yNjEgMTE4LjIzIDExNC41NThDMTE3LjkwMSAxMTQuODU1IDExNy41MDkgMTE1LjA5MSAxMTcuMDc5IDExNS4yNTJDMTE2LjY0OCAxMTUuNDEzIDExNi4xODYgMTE1LjQ5NiAxMTUuNzIgMTE1LjQ5NkMxMTQuNzc4IDExNS40OTYgMTEzLjg3NSAxMTUuMTU4IDExMy4yMSAxMTQuNTU4QzExMi41NDQgMTEzLjk1NyAxMTIuMTcgMTEzLjE0MyAxMTIuMTcgMTEyLjI5NEMxMTIuMTcgMTExLjQ0NSAxMTIuNTQ0IDExMC42MyAxMTMuMjEgMTEwLjAzQzExMy44NzUgMTA5LjQyOSAxMTQuNzc4IDEwOS4wOTIgMTE1LjcyIDEwOS4wOTJDMTE2LjE4NiAxMDkuMDkyIDExNi42NDggMTA5LjE3NSAxMTcuMDc5IDEwOS4zMzZDMTE3LjUwOSAxMDkuNDk2IDExNy45MDEgMTA5LjczMiAxMTguMjMgMTEwLjAzQzExOC41NiAxMTAuMzI3IDExOC44MjEgMTEwLjY4IDExOSAxMTEuMDY4QzExOS4xNzggMTExLjQ1NyAxMTkuMjcgMTExLjg3MyAxMTkuMjcgMTEyLjI5NFoiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyOC4xMSAxMTQuMTAzQzEyOC4xMSAxMTQuOTUzIDEyNy43MzYgMTE1Ljc2NyAxMjcuMDcgMTE2LjM2OEMxMjYuNDA0IDExNi45NjggMTI1LjUwMSAxMTcuMzA1IDEyNC41NiAxMTcuMzA1QzEyNC4wOTQgMTE3LjMwNSAxMjMuNjMyIDExNy4yMjMgMTIzLjIwMSAxMTcuMDYyQzEyMi43NzEgMTE2LjkwMSAxMjIuMzc5IDExNi42NjUgMTIyLjA1IDExNi4zNjhDMTIxLjcyIDExNi4wNyAxMjEuNDU4IDExNS43MTcgMTIxLjI4IDExNS4zMjlDMTIxLjEwMiAxMTQuOTQgMTIxLjAxIDExNC41MjQgMTIxLjAxIDExNC4xMDNDMTIxLjAxIDExMy42ODMgMTIxLjEwMiAxMTMuMjY2IDEyMS4yOCAxMTIuODc4QzEyMS40NTggMTEyLjQ5IDEyMS43MiAxMTIuMTM3IDEyMi4wNSAxMTEuODM5QzEyMi4zNzkgMTExLjU0MiAxMjIuNzcxIDExMS4zMDYgMTIzLjIwMSAxMTEuMTQ1QzEyMy42MzIgMTEwLjk4NCAxMjQuMDk0IDExMC45MDEgMTI0LjU2IDExMC45MDFDMTI1LjUwMSAxMTAuOTAxIDEyNi40MDQgMTExLjIzOSAxMjcuMDcgMTExLjgzOUMxMjcuNzM2IDExMi40NCAxMjguMTEgMTEzLjI1NCAxMjguMTEgMTE0LjEwM1oiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyMi4zMzMgMTE4Ljk3NkMxMjIuMzMzIDExOS44MjYgMTIxLjk1OCAxMjAuNjQgMTIxLjI5MyAxMjEuMjQxQzEyMC42MjcgMTIxLjg0MSAxMTkuNzI0IDEyMi4xNzggMTE4Ljc4MiAxMjIuMTc4QzExOC4zMTYgMTIyLjE3OCAxMTcuODU1IDEyMi4wOTYgMTE3LjQyNCAxMjEuOTM1QzExNi45OTMgMTIxLjc3NCAxMTYuNjAyIDEyMS41MzggMTE2LjI3MiAxMjEuMjQxQzExNS45NDMgMTIwLjk0MyAxMTUuNjgxIDEyMC41OSAxMTUuNTAzIDEyMC4yMDJDMTE1LjMyNCAxMTkuODEzIDExNS4yMzIgMTE5LjM5NyAxMTUuMjMyIDExOC45NzZDMTE1LjIzMiAxMTguNTU2IDExNS4zMjQgMTE4LjE0IDExNS41MDMgMTE3Ljc1MUMxMTUuNjgxIDExNy4zNjMgMTE1Ljk0MyAxMTcuMDEgMTE2LjI3MiAxMTYuNzEyQzExNi42MDIgMTE2LjQxNSAxMTYuOTkzIDExNi4xNzkgMTE3LjQyNCAxMTYuMDE4QzExNy44NTUgMTE1Ljg1NyAxMTguMzE2IDExNS43NzQgMTE4Ljc4MiAxMTUuNzc0QzExOS43MjQgMTE1Ljc3NCAxMjAuNjI3IDExNi4xMTIgMTIxLjI5MyAxMTYuNzEyQzEyMS45NTggMTE3LjMxMyAxMjIuMzMzIDExOC4xMjcgMTIyLjMzMyAxMTguOTc2WiIgZmlsbD0iIzU5Njg2RiIvPgo8cGF0aCBkPSJNMTE1LjMwOCAxMjEuOTA1QzExMy43MzUgMTIxLjQyNiAxMTUuNzA3IDEyMC42OCAxMTUuNDI5IDEyMC41NzNDMTE0LjY1MyAxMjAuMjc2IDExNS43MyAxMTkuMzMzIDExNi43NCAxMTguNTM5QzExNy42MjggMTE3Ljg0MSAxMTcuNjU5IDExNy44MzkgMTE3Ljg5MiAxMTguNDY4QzExOC4wMjQgMTE4LjgyMyAxMTguMzcxIDExOS4yMDYgMTE4LjY2NSAxMTkuMzE5QzExOS4yODkgMTE5LjU1OCAxMjAuOTUxIDExOS4yNjkgMTIwLjk1MSAxMTguODM1QzEyMC45NTEgMTE4LjIyMyAxMjEuNDE1IDExOC42OTkgMTIxLjc5NCAxMTkuNTMxQzEyMi40NTcgMTIwLjk4NyAxMjIuNDM3IDEyMi40NzIgMTIxLjQ1IDEyMi40NzJDMTIwLjg1OSAxMjIuNDcyIDEyMC4zMSAxMjIuNjkxIDEyMC4xOSAxMjMuMTQ3QzExNS4wNDYgMTI0LjYgMTE2LjQ3MSAxMjMuMjg0IDExNS4zMDggMTIxLjkwNVpNMTIzLjA5NCAxMTguMDU0QzEyMS42MDkgMTE3LjQ2NiAxMjAuNTQ3IDExNC44MzggMTIwLjU0NyAxMTMuMzA5QzEyMC41NDcgMTExLjMyMyAxMjIuNTQxIDEwOS4wOTUgMTI0LjMxNyAxMDkuMDk1QzEyOC4zMTUgMTA5LjA5NSAxMzAuNjg0IDExMi4yNjEgMTI4LjgzOCAxMTguNDA5QzEyOC4zODUgMTE5LjkxOSAxMjMuOTMzIDExOC4zODcgMTIzLjA5NCAxMTguMDU0Wk0xMjUuODY2IDExNS42NDlDMTI3LjU0NCAxMTQuNDc0IDEyNS44NTcgMTExLjc1NSAxMjMuOTgxIDExMi42MUMxMjIuNDc5IDExMy4yOTQgMTIzLjExOSAxMTYuMTE1IDEyNC43NyAxMTYuMTY0QzEyNC45NjEgMTE2LjE2OSAxMjUuNDU0IDExNS45MzggMTI1Ljg2NiAxMTUuNjQ5Wk0xMjQuMzM5IDExNC41NzlDMTIzLjc3MSAxMTQuMzgxIDEyMy44MDMgMTEzLjI1NiAxMjQuMzgzIDExMy4wMzRDMTI0LjkwMiAxMTIuODM1IDEyNS42MDQgMTEzLjM5OCAxMjUuNjA0IDExNC4wMTRDMTI1LjYwNCAxMTQuNDg5IDEyNC45MzYgMTE0Ljc4NyAxMjQuMzM5IDExNC41NzlaTTExMi4zMTkgMTE2Ljk4QzEwOS45NiAxMTUuNjcgMTEwLjI4NiAxMTEuMDA2IDExMi40MTcgMTA4Ljk2NUMxMTQuMzk2IDEwNy4wNjkgMTE4Ljc3OCAxMDcuMTAzIDExOS43NjQgMTA5LjQ2MkMxMjAuNDE1IDExMS4wMjEgMTIwLjIyOCAxMTIuNiAxMTkuMzk4IDExNC4wNzdDMTE4LjE5NCAxMTYuMjE5IDExNC4yNDIgMTE4LjA0OSAxMTIuMzE5IDExNi45OFpNMTE2LjU2IDExMy41OTNDMTE3LjMyNSAxMTIuOTAxIDExNy4yOTcgMTEyLjA2IDExNi41NzIgMTExLjI1OUMxMTUuNzc3IDExMC4zNzkgMTE0LjY5MiAxMTAuMzQ1IDExNC4wMzUgMTExLjM0N0MxMTMuNTI0IDExMi4xMjcgMTEzLjQzMSAxMTIuNTI4IDExMy45NDMgMTEzLjMwOUMxMTQuNTggMTE0LjI4IDExNS42NyAxMTQuMzk5IDExNi41NiAxMTMuNTkzWk0xMTQuMzc0IDExMi4zNEMxMTQuMTI4IDExMS42OTggMTE0Ljk4NSAxMTAuOTgxIDExNS42OCAxMTEuMjQ3QzExNS45NzQgMTExLjM2IDExNi4xNjQgMTExLjcxOCAxMTYuMTAyIDExMi4wNDRDMTE1Ljk2MSAxMTIuNzg1IDExNC42MzIgMTEzLjAxMiAxMTQuMzc0IDExMi4zNFoiIGZpbGw9IiNEM0Q2REEiLz4KPHBhdGggZD0iTTExOC40NjMgMTIxLjAwOEwxMTcuOTQ1IDEyMC43OEMxMTcuNTEgMTIwLjY4NSAxMTcuMjMxIDEyMS4xMDcgMTE3LjEzNiAxMjEuNTQzTDExNi44ODEgMTIyLjcwOUMxMTYuNzg2IDEyMy4xNDUgMTE3LjA2MSAxMjMuNTcxIDExNy40OTYgMTIzLjY2NkwxMTguNDQ3IDEyMy44NzRDMTE4Ljg4MiAxMjMuOTY5IDExOS4zMTEgMTIzLjY5NCAxMTkuNDA2IDEyMy4yNTlMMTE5LjY4NSAxMjEuNTU2QzExOS43OCAxMjEuMTIgMTE5LjQ3OSAxMjEuMjI4IDExOS4wNDMgMTIxLjEzM0wxMTguNjI4IDEyMS4wNDNDMTE4LjU3MyAxMjEuMzIxIDExOC41MTYgMTIxLjYxMyAxMTguNDcxIDEyMS44MzNDMTE4LjQzOCAxMjEuOTk4IDExOC40MDcgMTIyLjE0OCAxMTguMzc5IDEyMi4yODJDMTE4LjM1MSAxMjIuNDE1IDExOC4zMjcgMTIyLjUzMyAxMTguMzA1IDEyMi42MzVDMTE4LjI4MiAxMjIuNzM4IDExOC4yNjIgMTIyLjgyNSAxMTguMjQ1IDEyMi44OTdDMTE4LjIyOCAxMjIuOTY5IDExOC4yMTQgMTIzLjAyNSAxMTguMjAyIDEyMy4wNjhDMTE4LjE5NiAxMjMuMDg5IDExOC4xOTMgMTIzLjEwNyAxMTguMTg4IDEyMy4xMjFDMTE4LjE4MyAxMjMuMTM1IDExOC4xNzggMTIzLjE0NSAxMTguMTc1IDEyMy4xNTJDMTE4LjE3MyAxMjMuMTU1IDExOC4xNzIgMTIzLjE1NiAxMTguMTcxIDEyMy4xNThDMTE4LjE3IDEyMy4xNTkgMTE4LjE2OCAxMjMuMTYgMTE4LjE2NyAxMjMuMTZMMTE4LjE2NSAxMjMuMTU4QzExOC4xNjQgMTIzLjE1NiAxMTguMTYzIDEyMy4xNTIgMTE4LjE2MyAxMjMuMTQ4QzExOC4xNjIgMTIzLjE0IDExOC4xNjIgMTIzLjEyOSAxMTguMTYzIDEyMy4xMTVDMTE4LjE2MyAxMjMuMSAxMTguMTYzIDEyMy4wODMgMTE4LjE2NSAxMjMuMDYxQzExOC4xNjggMTIzLjAxOCAxMTguMTc1IDEyMi45NjEgMTE4LjE4MyAxMjIuODkxQzExOC4xOTIgMTIyLjgyIDExOC4yMDMgMTIyLjczNiAxMTguMjE2IDEyMi42NEMxMTguMjI5IDEyMi41NDMgMTE4LjI0NiAxMjIuNDMxIDExOC4yNjQgMTIyLjMwOEMxMTguMjgxIDEyMi4xODUgMTE4LjMwMSAxMjIuMDUxIDExOC4zMjMgMTIxLjkwM0MxMTguMzQ2IDEyMS43NTUgMTE4LjM3IDEyMS41OTIgMTE4LjM5NyAxMjEuNDE5QzExOC40MTYgMTIxLjI5OSAxMTguNDQyIDEyMS4xNCAxMTguNDYzIDEyMS4wMDhaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDMuNTgxIDExNi4zNjdDMTAzLjQ3OSAxMTYuMTAzIDEwMy42MjIgMTE1LjY5OSAxMDMuODk4IDExNS40NjlDMTA0LjMwMyAxMTUuMTMzIDEwNC41MDQgMTE1LjE1NSAxMDQuOTMgMTE1LjU4MUMxMDUuMjIgMTE1Ljg3MSAxMDUuMzU1IDExNi4yNzYgMTA1LjIzIDExNi40NzlDMTA0LjkwMiAxMTcuMDA5IDEwMy43OTggMTE2LjkzNCAxMDMuNTgxIDExNi4zNjdaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDExMi4yODFDMTA1LjY4IDExMS44NDkgMTA1LjcwMiAxMTEuNjYgMTA2LjE2NSAxMTEuMTk3QzEwNi42ODkgMTEwLjY3NCAxMDYuNzY0IDExMC42NzQgMTA3LjI4NyAxMTEuMTk3QzEwNy43NTEgMTExLjY2IDEwNy43NzMgMTExLjg0OSAxMDcuNDE1IDExMi4yODFDMTA3LjE3NiAxMTIuNTY4IDEwNi44NjYgMTEyLjgwMyAxMDYuNzI2IDExMi44MDNDMTA2LjU4NiAxMTIuODAzIDEwNi4yNzcgMTEyLjU2OCAxMDYuMDM4IDExMi4yODFaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDEwNy4yMjRDMTA1LjY4MiAxMDYuNzk1IDEwNS42OTQgMTA2LjYxMiAxMDYuMTA2IDEwNi4yMDFDMTA2LjY1IDEwNS42NTcgMTA3LjczOCAxMDUuODggMTA3LjczOCAxMDYuNTM2QzEwNy43MzggMTA2Ljk3IDEwNy4wNzIgMTA3Ljc0NyAxMDYuNyAxMDcuNzQ3QzEwNi41NzUgMTA3Ljc0NyAxMDYuMjc3IDEwNy41MTIgMTA2LjAzOCAxMDcuMjI0WiIgZmlsbD0iIzk4QzZEOCIvPgo8cGF0aCBkPSJNMTE3Ljk1NSAxMzYuNTQxQzExNC41NzggMTM2LjExMSAxMTAuMDg3IDEzNC44ODEgMTA5LjQxIDEzNC4yNzRDMTA4LjEyMyAxMzMuMTE5IDEwOC45NDIgMTI3Ljg3OSAxMTAuNDY2IDEyNi41NEMxMTEuMzg3IDEyNS43MzEgMTE1LjAzOSAxMjQuNjYzIDExNy4xODIgMTI1LjE2OUMxMTkuNjQyIDEyNS43NDkgMTIzLjkzMiAxMjguNjQ4IDEyNC4yNSAxMjkuNTA4QzEyNC42MjUgMTMwLjUyMiAxMjQuMDQ4IDEzNC41NDEgMTIzLjMyMyAxMzUuNTM2QzEyMi42OTQgMTM2LjM5OSAxMjAuMzI1IDEzNi44NDMgMTE3Ljk1NSAxMzYuNTQxWk0xMjAuNTQgMTM0LjA4NEMxMjEuMjk3IDEzMy4zOTkgMTIxLjM0MiAxMzEuODQyIDEyMC42MjcgMTMxLjEyNkMxMTkuNDkgMTI5Ljk4OSAxMTcuMTEyIDEzMC44NjkgMTE3LjExMiAxMzIuNDI4QzExNy4xMTIgMTM0LjMwNCAxMTkuMTg5IDEzNS4zMDcgMTIwLjU0IDEzNC4wODRaTTExOC41ODIgMTMzLjEyNUMxMTguMzExIDEzMi40MTkgMTE4LjY4IDEzMS42MDggMTE5LjI3MiAxMzEuNjA4QzEyMCAxMzEuNjA4IDEyMC4yMjggMTMyLjE3MyAxMTkuODEyIDEzMi45NDlDMTE5LjM4MSAxMzMuNzU1IDExOC44NTMgMTMzLjgzIDExOC41ODIgMTMzLjEyNVpNMTE2LjQyMiAxMzMuMTQzQzExNy4wOTQgMTMyLjMzMyAxMTYuNzExIDEzMS4xNzYgMTE1LjY4MSAxMzAuOTAyQzExNC41ODEgMTMwLjYxIDExNC4wNTIgMTMxLjE4MyAxMTQuODY5IDEzMS43OEMxMTUuNjgzIDEzMi4zNzUgMTE1LjU1MSAxMzIuNjc5IDExNC41MzMgMTMyLjU1N0MxMTMuMzI3IDEzMi40MTMgMTEyLjgzMyAxMzEuNTY2IDExMy4yNTQgMTMwLjM2MkMxMTMuNjgxIDEyOS4xNDIgMTE1LjE5NyAxMjguODYzIDExNS43NTMgMTI5LjkwMkMxMTYuMTkzIDEzMC43MjUgMTE2LjYyNyAxMzAuNzkzIDExNi45IDEzMC4wODNDMTE3LjIyOSAxMjkuMjI0IDExNS45MTQgMTI4LjIzNyAxMTQuNDQgMTI4LjIzN0MxMTEuOTkzIDEyOC4yMzcgMTEwLjgxNiAxMzEuMDc0IDExMi41NDYgMTMyLjgwM0MxMTMuNSAxMzMuNzU4IDExNS43NDkgMTMzLjk1NSAxMTYuNDIyIDEzMy4xNDNaIiBmaWxsPSIjN0JBOUI5Ii8+CjxwYXRoIGQ9Ik0xMjAuMTE4IDEzOC41OTJDMTE4Ljc1MSAxMzcuMjk3IDEyMS45MTYgMTM2LjA5NiAxMjMuNjA2IDEzOC4yODNDMTI0LjQ2OSAxMzkuMzIyIDEyMi44NTMgMTM5Ljk3NiAxMjAuMTE4IDEzOC41OTJaTTEwNi4xMiAxMzUuNjU4QzEwNC43NTEgMTM0LjI4OSAxMDYuOTYzIDEzMy45MDQgMTA4LjU4MSAxMzUuMjNMMTA5Ljk0MSAxMzUuOTJMMTA4LjA1OSAxMzYuMDYxQzEwNy4yMTUgMTM2LjA2MiAxMDYuMzQzIDEzNS44ODEgMTA2LjEyIDEzNS42NThWMTM1LjY1OFpNMTI1LjM2MyAxMjcuODY2QzEyNS4wODIgMTI3LjQxIDEyOC4zMzMgMTI2LjI2NyAxMjguNjg4IDEyNi40ODZDMTI4Ljg0NiAxMjYuNTg0IDEyOC45NzUgMTI3LjMzNyAxMjguOTc1IDEyOC4xNjFDMTI4Ljk3NSAxMjkuMjM5IDEyOC44NCAxMjkuNjU4IDEyOC40OSAxMjkuNjU4QzEyOC4yMjIgMTI5LjY1OCAxMjUuNDc2IDEyOC4wNDcgMTI1LjM2MyAxMjcuODY2Wk0xMTguNjQxIDEyMS4wOTVDMTE0Ljg3MSAxMjAuNzUzIDExNS4zNzggMTE5LjMyNyAxMTYuMzYxIDExOC40OTlDMTE2LjkyOCAxMTguMDIxIDExNy4zNCAxMTcuNzc5IDExNy45MTMgMTE3LjY2N0MxMTcuOTEzIDExNy42NjcgMTE3Ljg3MiAxMTkuMDQ2IDExOC42NjYgMTE5LjMxOUMxMTkuMjk3IDExOS41MzYgMTIwLjI3OCAxMTkuMDY0IDEyMC40NzEgMTE4LjY3NUMxMjIuMTg4IDExNS4yMDEgMTIxLjQxNSAxMTguNjk5IDEyMS43OTQgMTE5LjUzMkMxMjMuMjM4IDEyNC4wMDcgMTIwLjE4OCAxMjEuNzMgMTE4LjY0MSAxMjEuMDk1Wk0xMDQuOTEyIDEyMS4yM0MxMDQuMjQyIDEyMC40ODcgMTAzLjY5MyAxMTkuNzI5IDEwMy42OTMgMTE5LjU0NEMxMDMuNjkzIDExOS4xMTYgMTA0Ljc0NiAxMTkuMTEyIDEwNS44NjMgMTE5LjUzN0MxMDYuNTkzIDExOS44MTQgMTEwLjM0NyAxMjAuMDA2IDExMC4zNDcgMTIxLjE1M0MxMTAuMzQ3IDEyMS44OTkgMTA4Ljg5IDEyMy43NjIgMTA4LjcyNiAxMjMuNzYyQzEwOC41NjMgMTIzLjc2MiAxMDUuNTgzIDEyMS45NzIgMTA0LjkxMiAxMjEuMjNaIiBmaWxsPSIjQTZCM0MyIi8+CjxwYXRoIGQ9Ik0xMTguNDYzIDEyMS4wMDhDMTE4LjQ0MiAxMjEuMTQgMTE4LjQxNiAxMjEuMjk5IDExOC4zOTcgMTIxLjQxOUMxMTguMzcgMTIxLjU5MiAxMTguMzQ1IDEyMS43NTUgMTE4LjMyMyAxMjEuOTAzQzExOC4zIDEyMi4wNTEgMTE4LjI4MSAxMjIuMTg1IDExOC4yNjMgMTIyLjMwOEMxMTguMjQ1IDEyMi40MzEgMTE4LjIyOSAxMjIuNTQyIDExOC4yMTYgMTIyLjYzOUMxMTguMjAzIDEyMi43MzYgMTE4LjE5MSAxMjIuODIgMTE4LjE4MyAxMjIuODlDMTE4LjE3NSAxMjIuOTYxIDExOC4xNjggMTIzLjAxOCAxMTguMTY1IDEyMy4wNjFDMTE4LjE2MyAxMjMuMDgzIDExOC4xNjQgMTIzLjEgMTE4LjE2MyAxMjMuMTE1QzExOC4xNjIgMTIzLjEyOSAxMTguMTYyIDEyMy4xNCAxMTguMTYzIDEyMy4xNDhDMTE4LjE2MyAxMjMuMTUyIDExOC4xNjQgMTIzLjE1NiAxMTguMTY1IDEyMy4xNThMMTE4LjE2NyAxMjMuMTZDMTE4LjE2OCAxMjMuMTYgMTE4LjE3IDEyMy4xNTkgMTE4LjE3MSAxMjMuMTU4QzExOC4xNzIgMTIzLjE1NyAxMTguMTczIDEyMy4xNTUgMTE4LjE3NSAxMjMuMTUyQzExOC4xNzggMTIzLjE0NSAxMTguMTgyIDEyMy4xMzUgMTE4LjE4NyAxMjMuMTIxQzExOC4xOTIgMTIzLjEwNyAxMTguMTk2IDEyMy4wODkgMTE4LjIwMiAxMjMuMDY3QzExOC4yMTMgMTIzLjAyNSAxMTguMjI4IDEyMi45NjkgMTE4LjI0NSAxMjIuODk3QzExOC4yNjIgMTIyLjgyNSAxMTguMjgyIDEyMi43MzggMTE4LjMwNSAxMjIuNjM1QzExOC4zMjcgMTIyLjUzMyAxMTguMzUxIDEyMi40MTUgMTE4LjM3OSAxMjIuMjgxQzExOC40MDYgMTIyLjE0OCAxMTguNDM4IDEyMS45OTggMTE4LjQ3MSAxMjEuODMzQzExOC41MTYgMTIxLjYxMyAxMTguNTczIDEyMS4zMjEgMTE4LjYyOCAxMjEuMDQzTDExOC40NjMgMTIxLjAwOFoiIGZpbGw9IiM0QzlDQkIiLz4KPHBhdGggZD0iTTMwLjk5ODYgMTQyLjY3M0MyNi40NDg5IDE0MC45MTYgMjEuNDY2NCAxMzYuODA1IDE4Ljc5MjEgMTMyLjYwM0MxNS40MDUzIDEyNy4yODEgMTQuNDA1MiAxMjMuNDE5IDE1LjA2MzYgMTE4LjIwM0MxNS42ODA2IDExMy4zMTUgMTcuMzU1OCAxMTAuNTU3IDIyLjIyMzIgMTA2LjQxM0MyOS4wNDE3IDEwMC42MDggMzQuNDg0NiA5Ny45Nzk5IDQwLjU5NDcgOTcuNTQxNUM0OS44MjEyIDk2Ljg3OTQgNTUuMDk3OCA5OS44MTA5IDU3LjY0NjUgMTA3LjAxNUM1OC41MDQzIDEwOS40MzkgNTguNjg1NiAxMTUuNTcgNTguMDU0NCAxMjAuODA5QzU3LjgzMTEgMTIyLjY2MyA1Ny40Nzg5IDEyNi4wNTUgNTcuMjcxOCAxMjguMzQ4QzU2LjU0NjcgMTM2LjM3NSA1NC40MTQ3IDE0MC40MTcgNDkuNjI1MyAxNDIuODQzQzQ4LjI5NzkgMTQzLjUxNiA0Ny43MTk5IDE0My41NjEgNDAuNjkyNCAxNDMuNTM5QzMzLjUxNTEgMTQzLjUxNiAzMy4wODEgMTQzLjQ3OCAzMC45OTg2IDE0Mi42NzNaTTAuMDc5NTc2NSAxMDQuOTY1QzAuMDgxNTcyMyAxMDMuODUzIDAuMTQ0NDAzIDEwMy40MzggMC4yMTk0MjcgMTA0LjA0NEMwLjI5NDQ1MSAxMDQuNjUgMC4yOTI4OTkgMTA1LjU2IDAuMjE1OTk2IDEwNi4wNjdDMC4xMzkwNzkgMTA2LjU3MyAwLjA3NzY4MzggMTA2LjA3OCAwLjA3OTU3NjUgMTA0Ljk2NVpNMC4wMDE3MzM1MSAxMDIuMjRDMC4wMTc4OTkyIDEwMS44NDggMC4wOTc3ODk3IDEwMS43NjggMC4yMDUzOTggMTAyLjAzN0MwLjMwMjc3MiAxMDIuMjggMC4yOTA3OTcgMTAyLjU3MSAwLjE3ODc4NyAxMDIuNjgzQzAuMDY2Nzc0OCAxMDIuNzk1IC0wLjAxMjkwMjEgMTAyLjU5NiAwLjAwMTczMzUxIDEwMi4yNFpNMC4wMjgzNDQ0IDc1LjEzMjZDMC4wMjgzNDQ0IDc0LjY2OTEgMC4xMDQ4NTcgNzQuNDc5NSAwLjE5ODM2OSA3NC43MTEyQzAuMjkxODg0IDc0Ljk0MyAwLjI5MTg4NCA3NS4zMjIyIDAuMTk4MzY5IDc1LjU1MzlDMC4xMDQ4NTcgNzUuNzg1NyAwLjAyODM0NDQgNzUuNTk2MSAwLjAyODM0NDQgNzUuMTMyNloiIGZpbGw9InVybCgjcGFpbnQyX3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTM1LjMxMzMgMTEyLjU0NEwzNy45IDExMS4wOTFMMzkuMzUxNyAxMjEuNDZMMzkuMjQ4IDEyOC45MjZMMzUuMzcwMyAxMzAuODUxTDMyLjkyMjkgMTI2LjQzN0wzNS4zMTMzIDExMi41NDRaIiBmaWxsPSIjMzA2MEFEIi8+CjxwYXRoIGQ9Ik0zOC43Mjk1IDExNi42OUwzOS4zNTE2IDEyNC41NzFMNDMuMjkxOSAxMjUuOTE5TDQzLjM5NTYgMTA5LjIyNUw0MC4xODEyIDExMC4wNTRMMzguNzI5NSAxMTYuNjlaIiBmaWxsPSIjNjA2MzY4Ii8+CjxwYXRoIGQ9Ik0zNS4zNjk5IDEzMC44NTFDMzMuMDMyNCAxMjkuNzMgMjkuNzMwMSAxMjYuNDA4IDI4LjY2MjEgMTI0LjEwM0MyOC4wMzMyIDEyMi43NDUgMjcuNjM5NyAxMjIuMzEgMjYuODkyMyAxMjIuMTQ2QzI1Ljc4NzkgMTIxLjkwMyAyNS44MDg0IDEyMS45NzEgMjYuMzIxOCAxMjAuMjU4QzI2LjU5MzYgMTE5LjM1IDI2LjkwMzIgMTE4Ljk1NCAyNy4zNDA2IDExOC45NTRDMjcuNjg2MiAxMTguOTU0IDI5LjE1OTcgMTE3Ljg0MSAzMC42MTQ5IDExNi40OEMzMi4wNzAyIDExNS4xMTkgMzMuNzIyNiAxMTMuNjc3IDM0LjI4NjkgMTEzLjI3NUwzNS4zMTI5IDExMi41NDRMMzUuNjc2NyAxMTQuMjQxQzM3LjA0NDMgMTIwLjYxOCAzNy4wMjcgMTI0LjI2NiAzNS42MTk4IDEyNi4yNDNDMzQuNTUyMSAxMjcuNzQ0IDM0Ljg0MTcgMTI4LjM5MyAzNi41NzkxIDEyOC4zOTNDMzcuMzk3IDEyOC4zOTMgMzcuODIyMiAxMjguMTg1IDM4LjEzMDUgMTI3LjYzNEMzOC42NzQ4IDEyNi42NjMgMzguNjMxNSAxMjAuOTY4IDM4LjA0NTQgMTE2LjQyNkMzNy4zNDQ2IDExMC45OTUgMzcuMzEyNyAxMTEuMTY3IDM5LjE1NzggMTEwLjM5OEM0MC4wMzYgMTEwLjAzMSA0MC44MTggMTA5Ljc5NyA0MC44OTU1IDEwOS44NzdDNDAuOTczMSAxMDkuOTU2IDQwLjgxOTEgMTExLjIzNSA0MC41NTM0IDExMi43MThDMzkuNjA2NyAxMTguMDAzIDM5LjkzMTUgMTIzLjIzOCA0MS4yNzUyIDEyNC4zNTNDNDEuNjg1MSAxMjQuNjkzIDQxLjcyMzMgMTIzLjk2OSA0MS41NDUyIDExOS4yMzJDNDEuMzE1NiAxMTMuMTIzIDQxLjUxNzQgMTEwLjkxOSA0Mi40MDc1IDEwOS44MkM0Mi45MTk4IDEwOS4xODggNDMuMjEzIDEwOS4xMTYgNDQuNDk1OSAxMDkuMzFDNDYuOTU2NiAxMDkuNjgyIDQ2Ljk1MDYgMTA5LjY2OCA0Ni4yMDk5IDExMy4yNTRDNDUuODQ5NiAxMTQuOTk5IDQ1LjQ0ODMgMTE3LjIyNCA0NS4zMTggMTE4LjJMNDUuMDgxMiAxMTkuOTczTDQ1Ljk0NDYgMTE5LjY0NUM0Ni40MTk1IDExOS40NjUgNDcuNDQxMyAxMTguOTk0IDQ4LjIxNTMgMTE4LjU5OUw0OS42MjI2IDExNy44ODFMNDkuMzg5MyAxMTguNjdDNDkuMjYxIDExOS4xMDUgNDkuMDY3MiAxMjAuNzAxIDQ4Ljk1ODcgMTIyLjIxOEw0OC43NjE0IDEyNC45NzZMNDcuMTA5NSAxMjQuMzI1QzQ2LjIwMSAxMjMuOTY3IDQ1LjMxMTYgMTIzLjY3NCA0NS4xMzMxIDEyMy42NzRDNDQuOTQxIDEyMy42NzQgNDQuODA4NSAxMjUuNTMyIDQ0LjgwODUgMTI4LjIyNEM0NC44MDg1IDEzMS44OTEgNDQuNzE3MSAxMzIuNzc1IDQ0LjMzODYgMTMyLjc3NUM0MS4wMTk4IDEzMi4yOTUgMzguNzg0MiAxMzIuNDU4IDM1LjM2OTkgMTMwLjg1MVoiIGZpbGw9IiNFRUMyM0IiLz4KPHBhdGggZD0iTTQxLjE2MzkgMTMyLjQxOEM0MS4xNjM5IDEzMi4yODMgNDEuMzQwOCAxMzAuNTE1IDQxLjY2OTIgMTI4LjE0NUM0MS45OTc3IDEyNS43NzUgNDIuMzU3NyAxMjIuOTYxIDQyLjQ2OTIgMTIxLjg5M0M0Mi41ODA4IDEyMC44MjQgNDIuNzk2OSAxMjAuMDI3IDQyLjk0OTUgMTIwLjEyMUM0My42NDQzIDEyMC41NTEgNDQuMjkzOCAxMjkuMzc1IDQ0LjA0NzQgMTMxLjQyQzQzLjg5MDkgMTMyLjcxOSA0My42NTk1IDEzMi43MzcgNDMuMzAxOSAxMzIuNjc3QzQyLjY2MDIgMTMyLjU3IDQxLjkxMDEgMTMyLjUxMyA0MS4xNjM5IDEzMi40MThaTTMyLjU5NjkgMTIyLjczNUMzMS42ODEzIDEyMi41ODMgMzAuODI0MiAxMjEuNDQyIDMwLjgyNDIgMTIwLjM3NEMzMC44MjQyIDExOS40MDcgMzIuMjM4MSAxMTcuOTM3IDMzLjE2ODUgMTE3LjkzN0MzNC42NjY2IDExNy45MzcgMzUuODM5MiAxMjAuNTEzIDM0Ljk0ODQgMTIxLjg0N0MzNC41NTE3IDEyMi40NDEgMzMuNDE0NyAxMjIuODcgMzIuNTk2OSAxMjIuNzM1WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTExNC4wMjEgMTM3LjQ5OEMxMDcuMzE4IDEzNS43NCAxMDYuMjczIDEzNC4xMDQgMTA2LjYwOSAxMjUuODkxQzEwNi43MTYgMTIzLjI4MSAxMDYuODg1IDEyMC41MzkgMTA2Ljk4NiAxMTkuNzk3QzEwNy41NDUgMTE1LjY2NCAxMDguMzQyIDExMi4xNzMgMTA5LjA4NiAxMTAuNTk5QzEwOS43NDQgMTA5LjIwNiAxMDkuODQyIDEwOC43MTIgMTA5LjUzNSAxMDguMzQyQzEwOC40NSAxMDcuMDM1IDExMC4zNzEgMTA0Ljk5NSAxMTEuOTEgMTA1LjgxOUMxMTIuNDU3IDEwNi4xMTEgMTEyLjkxOCAxMDYuMDc3IDExMy45NDUgMTA1LjY2NUMxMTcuMDYyIDEwNC40MTkgMTIzLjExNyAxMDUuNTcxIDEyNi44MTQgMTA4LjExNUMxMjguMTQ5IDEwOS4wMzMgMTI4LjQ4NSAxMDkuMTM2IDEyOS4wMzggMTA4Ljc5MUMxMzAuNTIzIDEwNy44NjMgMTMyLjQ1MSAxMTAuNTczIDEzMS4yNSAxMTEuOUMxMzAuNzg3IDExMi40MTIgMTMwLjc1NSAxMTIuNzA1IDEzMS4wNjUgMTEzLjU5NkMxMzEuODUgMTE1Ljg0NiAxMzEuNzk0IDExNi4xMDQgMTI4LjAwNiAxMjcuNzE5QzEyNS41OCAxMzUuMTYgMTI0LjU0MyAxMzcuMzY1IDEyMy4yNjQgMTM3LjgxMUMxMjEuNjUzIDEzOC4zNzMgMTE2LjcyMyAxMzguMjA2IDExNC4wMjEgMTM3LjQ5OFpNMTIyLjU0MSAxMzUuMjI5QzEyMy4wMDMgMTM0LjU0MSAxMjMuMzU5IDEzMy4zMTMgMTIzLjUxMyAxMzEuODc4QzEyMy43OTQgMTI5LjI1IDEyMy44MTkgMTI5LjI4MyAxMjAuMzE3IDEyNy42MDFDMTE3LjY4NSAxMjYuMzM4IDExMy44MzMgMTI1Ljk3MiAxMTEuODU1IDEyNi43OThDMTEwLjgyOSAxMjcuMjI3IDExMC41OTUgMTI3LjU1NiAxMTAuMDU5IDEyOS4zMjdDMTA5LjExNiAxMzIuNDQzIDEwOS43MDMgMTM0LjMwOCAxMTEuODMyIDEzNC45NThDMTEzLjEzOSAxMzUuMzU3IDEyMC4wOSAxMzYuNDMgMTIwLjk4IDEzNi4zN0MxMjEuNTA0IDEzNi4zMzUgMTIyLjA4MiAxMzUuOTEyIDEyMi41NDEgMTM1LjIyOVpNMTE5LjQ1MSAxMjIuNDJDMTE5LjYzIDEyMi4wODEgMTE5LjY4NSAxMjEuNTU2IDExOS42ODUgMTIxLjU1NkMxMjAuMTU2IDEyMS42NTQgMTIwLjQ4IDEyMS42ODMgMTIwLjcyNiAxMjEuNzE5QzEyMS4xMDUgMTIxLjc3NCAxMjEuNTY0IDEyMS41ODcgMTIxLjc0NSAxMjEuMzAzQzEyMi4xNDkgMTIwLjY3MSAxMjAuNDc0IDExNy43MDEgMTE5LjU0OCAxMTcuNDA4QzExOC42NjggMTE3LjEyOCAxMTUuOTM1IDExOC44MjcgMTE1LjkzNSAxMTkuNjUzQzExNS45MzUgMTIwLjAyMiAxMTYuNTUzIDEyMC40MTEgMTE2LjgzMSAxMjAuNTE4QzExNi45ODIgMTIwLjU3NiAxMTcuMjUyIDEyMC43MzggMTE3LjU3NiAxMjAuOTM4QzExNy4zOCAxMjEuMzUxIDExNy4zNDYgMTIxLjQ3OCAxMTcuMzEyIDEyMS43MjRDMTE3LjIwNCAxMjIuNDk4IDExNy40MzUgMTIyLjY4NiAxMTguNDE2IDEyMi45MTJDMTE5LjE1MSAxMjIuOTkxIDExOS4yNSAxMjIuODQ1IDExOS40NTEgMTIyLjQyWk0xMjcuMjY1IDExNi44MzZDMTMwLjkzMyAxMTMuNDc1IDEyNy4wNzQgMTA3LjQxIDEyMi43MDUgMTA5LjY3QzEyMS42MjYgMTEwLjIyOCAxMjAuNjU0IDExMi4wMTkgMTIwLjY1NCAxMTMuNDQ5QzEyMC42NTQgMTE1LjY2NCAxMjIuODMxIDExNy45MzcgMTI0Ljk1NyAxMTcuOTQxQzEyNS43MzMgMTE3Ljk0MyAxMjYuNDE0IDExNy42MTcgMTI3LjI2NSAxMTYuODM2Wk0xMTcuMTQxIDExNS42MjJDMTE5LjAzNSAxMTQuNDY4IDExOS44MDQgMTExLjc5NSAxMTguODAyIDEwOS44NTdDMTE4LjA4MiAxMDguNDY0IDExNy4wMDggMTA3LjgzIDExNS4zNjYgMTA3LjgzQzExMS41MDggMTA3LjgzIDEwOS40ODUgMTEzLjIzNCAxMTIuNDY1IDExNS41NzhDMTEzLjU3IDExNi40NDggMTE1Ljc1NCAxMTYuNDY4IDExNy4xNDEgMTE1LjYyMloiIGZpbGw9IiMwOTk2RDEiLz4KPHBhdGggb3BhY2l0eT0iMC45IiBkPSJNMzMuMTc1NyAyNkwzMi4zODEgMjcuMjE5QzMxLjk0NDkgMjcuODg5IDMxLjI4MzYgMjkuMzI0OCAzMC45MDkxIDMwLjQwODhDMzAuNTM0NyAzMS40OTI4IDI5Ljk5MjIgMzIuOTcwMSAyOS43MDQzIDMzLjY5MjhDMjkuNDE2MyAzNC40MTU0IDI5LjE3MzMgMzYuMTE1IDI5LjE2MzggMzcuNDdDMjkuMTQzNiA0MC4zNTMgMjguNzQ5NCA0MS44ODg4IDI3LjYzNjQgNDMuNDIyNUMyNi45MjEyIDQ0LjQwOCAyNi42NzUxIDQ0LjUyMjMgMjUuNDUzMiA0NC40NDFDMjMuODk5NSA0NC4zMzc3IDIyLjgxNTMgNDUuMDgxNSAyMS4yMTcgNDcuMzQ4MUMyMC43MTY1IDQ4LjA1OCAyMC4xMzY3IDQ4LjYzOTMgMTkuOTI4OSA0OC42MzkzQzE5LjcyMTEgNDguNjM5MyAxOS4xODA1IDQ4LjIzMTkgMTguNzI2MiA0Ny43MzUxQzE4LjI3MTkgNDcuMjM4MiAxNy45OTQ0IDQ3LjAxMTUgMTguMTExIDQ3LjIyOThDMTguNDkgNDcuOTM5NiAxNy41OTA0IDUzLjI2MTUgMTYuODM3OCA1NC43NjQyQzE1Ljk2NyA1Ni41MDI4IDE0LjM4NzIgNTcuNzIzMyAxMi41MzU0IDU4LjA4NDNDMTEuMDA1OSA1OC4zODI2IDExLjIyODMgNTguMDkgOC43MTc5NSA2My4wOTQ2QzYuNjYzMzQgNjcuMTkwNyA1Ljk2OTM1IDY3LjgyNjQgMi42NTUzMyA2OC42NTQyQzEuNDk3MTcgNjguOTQzNCAwLjQ1ODc1OSA2OS43MTQyIDAgNzAuNTI4N1Y3Mi44MjYzQzAuMzU2MTkzIDcyLjc0NzMgMC44Mjg3MjggNzIuNjIwMyAxLjYxNDk5IDcyLjM4NzNDMi45OTQ4NiA3MS45NzgyIDQuODIzNDggNzEuNDgwNiA1LjY3ODEgNzEuMjgwNkM3LjcyOTM0IDcwLjgwMDYgOS4zOTM1MSA2OS4yMTk1IDkuOTIyNzkgNjcuMjQ2N0MxMC4yNDI0IDY2LjA1NTQgMTAuNzAzMyA2NS40MjQxIDExLjk5OTIgNjQuNDA1OEMxMy4yODQgNjMuMzk2MiAxMy43NTY1IDYyLjc1NDYgMTQuMDY3MSA2MS41OTY5QzE0LjM3MzkgNjAuNDUzMiAxNC43MDExIDYwLjAwNCAxNS40NDA3IDU5LjcwNjNDMTcuMDE1MyA1OS4wNzI0IDIwLjY0NDQgNTUuODY1NSAyMS4yMzYzIDU0LjU4NThDMjEuNTM3MyA1My45MzQ5IDIyLjExOTggNTIuNTM5OSAyMi41Mjg3IDUxLjQ4NjJDMjMuMjYzNCA0OS41OTMyIDI1LjAxOTUgNDcuMzI2MSAyNS43NTAxIDQ3LjMyNjFDMjUuOTU3NCA0Ny4zMjYxIDI3LjEwOSA0Ni43MjQ4IDI4LjMwOTMgNDUuOTkwOEMzMS4yMzc2IDQ0LjIwMDMgMzEuNTk5MSA0My40MTM2IDMxLjU5OTEgMzguODYxNEMzMS41OTkxIDM1LjY4MTQgMzEuNjg3IDM1LjE1NTcgMzIuMzQ2OCAzNC4zNjg0QzMzLjAwMzcgMzMuNTg0NyAzMy4wOTk3IDMzLjAyNDkgMzMuMTM1MSAyOS43MzkxTDMzLjE3NTcgMjZaTTI0LjMzMzggNjMuNTk1OEMyNC4yMDM3IDYzLjYxMzUgMjQuMDczOCA2My42NjE4IDIzLjkyNTggNjMuNzM2MkMyMi45Mzk3IDY0LjIzMTUgMjIuNjEzNiA2NC45MTAyIDIxLjU5NzMgNjguNTg2QzIxLjAxNzkgNzAuNjgxNSAyMC40MjEyIDcyLjA4MTkgMTkuODc5OCA3Mi42MTc4QzE5LjQyMzUgNzMuMDY5NSAxOC45MzgzIDczLjg4MjMgMTguODAxIDc0LjQyNDJDMTguMTgxIDc2Ljg3MTUgMTYuNzQ1NCA3OC42MDczIDE0LjQ0OTUgNzkuNjg3MUMxMi40NTU1IDgwLjYyNDkgMTAuNjQwNyA4Mi4xMTc5IDkuMTExMDIgODQuMDc5OEM4LjMyMTA5IDg1LjA5MjkgNy40ODc1MiA4Ni4wMjkgNy4yNTY3OCA4Ni4xNjA5QzYuNzYxNjQgODYuNDQ0IDUuNzA4OTkgODguODk0MSA0LjYwNzg1IDkyLjMyOEMzLjcxMDkyIDk1LjEyNSAxLjcxMDY1IDk3LjQ2ODIgMC40Nzg1MTQgOTcuMTY1OEMwLjI1NDkzMSA5Ny4xMTA5IDAuMTA2OTExIDk3LjExMzcgMCA5Ny4yOTAxVjEwNC40OTJDMC41NTY0MTEgMTAzLjY2NCAxLjIzNDIgMTAyLjYyOSAyLjU2MzQ3IDEwMC41NEM0LjM0NTExIDk3LjczOTcgNi40NjM4NSA5NC40ODY3IDcuMjcxNzMgOTMuMzEyNEM4LjA3OTYgOTIuMTM4MSA5LjM1Njg3IDkwLjE0ODggMTAuMTA4NiA4OC44OTE2QzExLjIzNDIgODcuMDA5MiAxMi4xMDE2IDg2LjEyNDMgMTUuMDI4NCA4My44ODMzTDE4LjU4MzEgODEuMTYyN0wyMC40MDUzIDc1Ljk4ODFDMjMuMTg5NSA2OC4wODE3IDIzLjk1ODUgNjcuMTk5NyAyNy43NjI0IDY3LjU1MTVDMjkuNDYwNSA2Ny43MDg1IDMwLjYzMzQgNjcuNjE5NiAzMi4wNzM0IDY3LjIyNjdDMzQuMTgyNiA2Ni42NTEyIDM0LjQ3OTkgNjYuMTg3MiAzMy4zNTUxIDY1LjIzMThDMzIuNjU1NCA2NC42Mzc2IDMxLjMxNTMgNjQuNjQgMjguMjM4OCA2NS4yNDE4QzI3LjMzOTcgNjUuNDE3NyAyNi45MzQzIDY1LjI3NTcgMjUuODkzMiA2NC40MTc4QzI1LjEwNTEgNjMuNzY4NCAyNC43MjQxIDYzLjU0MjYgMjQuMzMzOCA2My41OTU4Wk0zMS43MTY2IDgzLjI1OThDMzEuNjYwNSA4My4yNzQ5IDMxLjU5MjEgODMuMzEyNiAzMS41MTE1IDgzLjM3MjFDMzEuMjcwOSA4My41NSAzMC42NTM2IDgzLjc5IDMwLjEzNzkgODMuOTA1NEMyOS4zODc2IDg0LjA3MzQgMjkuMDQ5OCA4NC40OTMyIDI4LjQ0ODIgODYuMDA0NUMyOC4wMzQ3IDg3LjA0MzIgMjcuNDkxNSA4OC4xMTUzIDI3LjI0MTIgODguMzg2M0MyNi45OTA5IDg4LjY1NzMgMjYuNDc1IDg5LjU1MSAyNi4wOTQxIDkwLjM3MzJDMjQuOTQyMiA5Mi44NTkyIDIzLjUyODIgOTQuMDYxOCAxOS45MDExIDk1LjY0MjFDMTYuNjEwNyA5Ny4wNzU2IDE2LjU2NyA5Ny4xMDg2IDE1LjY2NzEgOTguOTgwMkMxNS4xMDI3IDEwMC4xNTQgMTQuNjc5MiAxMDEuNzU3IDE0LjU0OTkgMTAzLjIwNUMxNC4yNzMxIDEwNi4zMDIgMTMuNTk2MiAxMDcuMDMxIDkuNTIxMTggMTA4LjYxNkM1LjkxNDk3IDExMC4wMTkgMy41NzcwMiAxMTEuNzQ1IDMuNDE1ODMgMTEzLjEyM0MzLjI0MDQxIDExNC42MjMgMi41OTgxMyAxMTUuMDg0IDAuOTI5MjYgMTE0LjkwN0MwLjUzOTUyOSAxMTQuODY2IDAuMjQ0MzMyIDExNC44NTEgMCAxMTQuODgxVjEyMUMxLjAyNjU4IDExOS42NTggMi43MTkyMiAxMTguMzYxIDYuMjI0OTcgMTE2LjIwNkMxMC4zNjA0IDExMy42NjUgMTIuNzA2NSAxMTEuNTc1IDEzLjU3NzkgMTA5LjY1NEMxMy45MjYgMTA4Ljg4NyAxNC42NTQ3IDEwNy4zOTMgMTUuMTk3MSAxMDYuMzM0QzE1LjczOTUgMTA1LjI3NSAxNi4yODQ1IDEwMy45MDEgMTYuNDA4NCAxMDMuMjgxQzE2LjczMjQgMTAxLjY2IDE4LjMwMzIgMTAwLjM0OSAyMC42NjU5IDk5LjcyNkMyNS4xNzcgOTguNTM3MiAyNS43NDYxIDk4LjAxNjMgMjYuMzUyNSA5NC41MzU0QzI2Ljg1NjEgOTEuNjQ1MiAyOC42MTIgODguMDY2MSAzMC41MjY3IDg2LjAyNDZDMzEuMzA5NSA4NS4xOSAzMS45NDk1IDg0LjE4IDMxLjk0OTUgODMuNzc5MUMzMS45NDk1IDgzLjM3NjYgMzEuODg0OSA4My4yMTQ2IDMxLjcxNjYgODMuMjU5OFoiIGZpbGw9IiMzNTkxMzYiLz4KPHBhdGggZD0iTTguMDgxNzcgNzQuNDA4M0M2LjAzNDE5IDczLjg3MDUgNS44OTQ0MyA3My43MzAyIDYuMDQyNzEgNzIuMzYxNUM2LjEzMDI1IDcxLjU1MzMgNi4yNDAxNyA3MC44NTc1IDYuMjg2OTYgNzAuODE1NEM2LjMzMzc2IDcwLjc3MzMgNy43MDQ0MyA3MS4wNjg3IDkuMzMyOTIgNzEuNDcxOUMxMS45NzY4IDcyLjEyNjQgMTIuNDg2NiA3Mi4xMjM4IDE0LjA5NDcgNzEuNDQ3NEMxNi40MDQxIDcwLjQ3NiAxNy4yMzgzIDY4Ljc3NTMgMTYuNDAyMiA2Ni43NDMxQzE1Ljk0MjQgNjUuNjI1NyAxNC45MjA0IDY0LjgyODEgMTIuMjM4NCA2My40OTRDNy42NDE3IDYxLjIwNzUgNi41MzA2MiA1OS45OTkgNi4yNzQ2NSA1Ny4wMDc2QzUuODkzMzcgNTIuNTUxOSA4Ljg0Njg5IDQ5Ljk4MzYgMTQuMzUyMiA0OS45ODM2QzE1Ljk3MzMgNDkuOTgzNiAxNy44NDg3IDUwLjE5MzcgMTguNTE5NiA1MC40NTA1QzE5LjYzNjggNTAuODc4MSAxOS42OTM3IDUxLjAzOTkgMTkuMTk0NSA1Mi4zNzEyQzE4LjcxODIgNTMuNjQxNiAxOC41MDI3IDUzLjc2ODggMTcuNDg2MSA1My4zNzk2QzE1LjUxOTkgNTIuNjI3IDEyLjE5NDggNTIuODU1NCAxMS4wMDU4IDUzLjgyNDdDOS43OTA1OCA1NC44MTUzIDkuNTU3MzYgNTYuOTI1MiAxMC41MzI0IDU4LjEwNzlDMTAuODcyNyA1OC41MjA3IDEyLjkwODcgNTkuNzc5IDE1LjA1NjkgNjAuOTA0MUMxOC40NjI4IDYyLjY4ODEgMTkuMDgyIDYzLjIyMTQgMTkuODk1MSA2NS4wNzE5QzIxLjExNTIgNjcuODQ4NyAyMC42Nzg4IDcwLjM0MzcgMTguNjA3OCA3Mi40MzIzQzE2LjE2MDcgNzQuOTAwMiAxMi41NDEzIDc1LjU3OTYgOC4wODE3NyA3NC40MDgzWk0yOC45MzI4IDc0LjM4MjNDMjUuMjM2NSA3Mi45OTU4IDIzLjcxMzggNzAuNTQwNSAyMy43MTM4IDY1Ljk2NjdDMjMuNzEzOCA2Mi41MDM0IDI0LjU1ODggNjAuNDY5NyAyNi44MzMzIDU4LjQ1OTNDMjguMzk2NCA1Ny4wNzc2IDI4Ljk4NjcgNTYuODY4NiAzMS4zMjY4IDU2Ljg2ODZDMzIuODE2OCA1Ni44Njg2IDM0LjY4MjEgNTcuMjEyOSAzNS41MDA1IDU3LjYzODlDMzcuMzE2OSA1OC41ODQ2IDM4LjgzMTkgNjEuNzQyNiAzOC44NDY2IDY0LjYxNDNMMzguODU3MyA2Ni43MDQ0SDMyLjk0MjNIMjcuMDI3M0wyNy4zNDIgNjguMDU2OUMyNy41MTUgNjguODAwNyAyOC4zMzU0IDcwLjAzOTEgMjkuMTY1IDcwLjgwOUMzMC41NzczIDcyLjExOTYgMzAuODk0OSA3Mi4xOTc5IDM0LjE1NDcgNzIuMDM4NUMzNy4zOTM4IDcxLjg4MDEgMzcuNjQ2MyA3MS45NDA5IDM3Ljc4NCA3Mi45MTIyQzM3Ljk2MjYgNzQuMTcyNSAzNy4xODgyIDc0LjUyMzYgMzMuNTI0MiA3NC44NDM0QzMxLjgwODUgNzQuOTkzMiAzMC4xMDU2IDc0LjgyMjIgMjguOTMyOCA3NC4zODIzWk0zNS4wODgxIDYyLjUyNDJDMzQuOTU1OCA2MC42NzMgMzMuMzY0NCA1OS4zMjc2IDMxLjMwNzIgNTkuMzI3NkMyOS43MDg2IDU5LjMyNzYgMjguNzI4MSA2MC4wOTA5IDI3LjY2MiA2Mi4xNjU1QzI2Ljc0NTcgNjMuOTQ4NiAyNy40MTIgNjQuMjk2NSAzMS40NDg4IDY0LjE0MjVMMzUuMTkzNiA2My45OTk2TDM1LjA4ODEgNjIuNTI0MlpNNDQuMzkxNCA3NC4zNzA4QzQyLjU3MTEgNzMuNTQ2OCA0MS43MDU5IDcyLjEyODYgNDEuNjQyMSA2OS44NjQ3QzQxLjU0MTYgNjYuMjk0NCA0NC4xMTEgNjQuMTQ2OCA0OS4yNTc2IDYzLjQ5OTRDNTEuODQxNCA2My4xNzQ0IDUyLjA0OTUgNjMuMDU3NyA1MS43NTkzIDYyLjA5NjlDNTAuOTY1IDU5LjQ2NjYgNDguMjQyMyA1OC41NjQ0IDQ0LjkyNzYgNTkuODMzMUM0My40MzAzIDYwLjQwNjMgNDMuNDg2OCA2MC40MzEzIDQzLjA1MTQgNTkuMDAzMkM0Mi43ODAxIDU4LjExMzUgNDIuOTk4MiA1Ny44NTExIDQ0LjM5NSA1Ny4zODdDNDYuOTM2NCA1Ni41NDI3IDUxLjM3NDUgNTYuNjM0MiA1Mi43Nzc5IDU3LjU2QzU0LjkzOTIgNTguOTg1NyA1NS4zNzUxIDYwLjYwMTkgNTUuNTQ4NyA2Ny44MzI4TDU1LjcxMDYgNzQuNTc0NUw1NC4xMjMgNzQuNTc0QzUzLjAzNzcgNzQuNTczMyA1Mi41MzUzIDc0LjMzOTYgNTIuNTM1MyA3My44MzU0QzUyLjUzNTMgNzIuOTA2NiA1Mi4wNTI4IDcyLjkwMjQgNTAuNzUyIDczLjgxOTZDNDkuMjAxMyA3NC45MTMgNDYuMTY5NyA3NS4xNzU3IDQ0LjM5MTQgNzQuMzcwOFpNNTAuOTQ3NyA3MC44MTc1QzUxLjcwOTkgNjkuOTczNSA1Mi4wNDY4IDY4Ljk4NiA1Mi4wNDY4IDY3LjU5NjZWNjUuNTkyOUw0OS44MTgyIDY1Ljg0NThDNDUuNzE5IDY2LjMxMSA0My44NzQ5IDY5LjUxMTggNDYuNTA1OCA3MS41OTUzQzQ4LjAzNzEgNzIuODA3OSA0OS4zNTk0IDcyLjU3NjMgNTAuOTQ3NyA3MC44MTc1Wk05Mi4wMDIgNzQuNTc4OEM4OC40MDI0IDczLjQ4NjQgODYuMjQxOSA3MC4yNDY2IDg2LjI0MTkgNjUuOTQxQzg2LjI0MTkgNjAuNDA0MiA4OS41MjExIDU2LjgzMjUgOTQuNjQxIDU2Ljc5MjVDOTguNjk4OSA1Ni43NjA5IDEwMS4yNjMgNTkuNDY3OSAxMDEuNTAzIDY0LjAzNjZMMTAxLjYzIDY2LjQ1ODVMOTYuMjU2MiA2Ni41ODg1QzkzLjMwMDcgNjYuNjYgOTAuNjE1OCA2Ni44MTYgOTAuMjg5NyA2Ni45MzUxQzg5LjM4MTggNjcuMjY2OCA5MC40Nzg4IDY5Ljk0NyA5Mi4wMzA2IDcxLjE4ODZDOTMuMTU5OSA3Mi4wOTIxIDkzLjc3MSA3Mi4xOTEyIDk2LjgxNzggNzEuOTY1QzEwMC4xOTMgNzEuNzE0NCAxMDAuMzE4IDcxLjc0NDkgMTAwLjUzMSA3Mi44Njg0QzEwMC43MyA3My45MTMzIDEwMC41MDkgNzQuMDgzNSA5OC4zNTc5IDc0LjU0OEM5NS42MTAzIDc1LjE0MTIgOTMuODgyOSA3NS4xNDk2IDkyLjAwMiA3NC41Nzg4Wk05Ny45NjU5IDYzLjA4MjNDOTcuOTY1OSA2MS4xMjMxIDk2LjE4NTggNTkuMzI3NiA5NC4yNDM0IDU5LjMyNzZDOTIuMjQ5NCA1OS4zMjc2IDkwLjE0OTkgNjEuMjU2MiA5MC4xNDk5IDYzLjA4NzhDOTAuMTQ5OSA2NC4yMDcgOTAuMjc5NyA2NC4yNDU1IDk0LjA1NzkgNjQuMjQ1NUM5Ny44NDE0IDY0LjI0NTUgOTcuOTY1OSA2NC4yMDg0IDk3Ljk2NTkgNjMuMDgyM1pNMTEwLjA0MiA3NC40N0MxMDYuMTQ5IDczLjQzIDEwMy45MyA2OS40MDI5IDEwNC41NCA2NC40ODYyQzEwNS4xNjYgNTkuNDQ0MiAxMDcuOTUxIDU2Ljg3MzYgMTEyLjc5MiA1Ni44NzA1QzExNS4zNzQgNTYuODY5IDExNS44MDUgNTcuMDI0OSAxMTcuMjYgNTguNDlDMTE4Ljg3OCA2MC4xMTg1IDExOS45NDEgNjIuODM0MyAxMTkuOTQ2IDY1LjM1MkwxMTkuOTQ5IDY2LjcwNDRIMTE0LjA4N0MxMDguNDg4IDY2LjcwNDQgMTA4LjIyNSA2Ni43NTAxIDEwOC4yMjUgNjcuNzIwM0MxMDguMjI1IDcwLjcxMjIgMTEwLjk1MiA3Mi4zNDQyIDExNS4zNTcgNzEuOTg4MkMxMTguMTM2IDcxLjc2MzcgMTE4LjQyMyA3MS44MzE1IDExOC42NjcgNzIuNzY3NEMxMTguOTk4IDc0LjA0NDMgMTE4Ljk2OCA3NC4wNzIgMTE2LjY5IDc0LjYwNDNDMTE0LjM2NyA3NS4xNDcgMTEyLjQzIDc1LjEwNzggMTEwLjA0MiA3NC40N1pNMTE2LjExOSA2Mi43NzAxQzExNS44NjQgNjAuODc4NyAxMTQuMTU5IDU5LjMyNzYgMTEyLjMzNSA1OS4zMjc2QzExMC41NjYgNTkuMzI3NiAxMDguMjI0IDYxLjYwMTQgMTA4LjIyNCA2My4zMTk1QzEwOC4yMjQgNjQuMjEyOCAxMDguNTI3IDY0LjI3NDYgMTEyLjI1NSA2NC4xNDI0QzExNi4yNDEgNjQuMDAxMSAxMTYuMjgzIDYzLjk4NjIgMTE2LjExOSA2Mi43NzAxWk0xMjYuODk0IDc0LjI0N0MxMjQuODI2IDczLjIyMDggMTIzLjUzMiA3MS4zNTg5IDEyMi44OTQgNjguNDg4OUMxMjEuOTc1IDY0LjM2MDUgMTIzLjA3NCA2MC40OTE0IDEyNS44MyA1OC4xNTY3QzEyNy44MjcgNTYuNDY1NyAxMzEuODM4IDU2LjMzMDUgMTMzLjgwNSA1Ny44ODc5TDEzNS4wOTIgNTguOTA3MlY1My45NTM2VjQ5SDEzNy4wNDZIMTM5VjYxLjc4NjVWNzQuNTczMUgxMzcuMjlDMTM1Ljc2MiA3NC41NzMxIDEzNS41OCA3NC40MzMxIDEzNS41OCA3My4yNTc5VjcxLjk0MjdMMTM0LjQ4MSA3Mi45NzkyQzEzMi4yMzMgNzUuMDk5OCAxMjkuNTI1IDc1LjU1MjMgMTI2Ljg5NCA3NC4yNDdaTTEzMy42NjQgNzAuNjc2NkMxMzQuOTc2IDY5LjM1NTcgMTM1LjA5MiA2OC45NTM3IDEzNS4wOTIgNjUuNzIwOUMxMzUuMDkyIDYyLjQ1NzIgMTM0Ljk4NSA2Mi4wOTQ1IDEzMy42MDcgNjAuNzA3NkMxMzIuMzY1IDU5LjQ1NzUgMTMxLjg2NSA1OS4yNjIyIDEzMC41NTQgNTkuNTE1NEMxMjYuMTQ0IDYwLjM2NzIgMTI0LjgzNSA2OC4xNDIzIDEyOC41OTggNzEuMTIyOUMxMzAuMzQ4IDcyLjUwODEgMTMxLjk4OCA3Mi4zNjM1IDEzMy42NjQgNzAuNjc2NlpNNjEuNDEwNyA2Ni4wNTg0QzYwLjAyMiA2MS4zNzU0IDU4Ljg4NTkgNTcuMzkxOSA1OC44ODU5IDU3LjIwNjJDNTguODg1OSA1Ny4wMjA2IDU5LjcyNzQgNTYuODY4NiA2MC43NTU5IDU2Ljg2ODZINjIuNjI1OUw2NC4yNTUxIDYzLjg3NjZDNjUuMTUxMiA2Ny43MzEgNjUuOTU2IDcwLjY2MzMgNjYuMDQzNyA3MC4zOTI5QzY2LjEzMTMgNzAuMTIyNCA2Ny4wNTU3IDY3LjAyNDEgNjguMDk3OCA2My41MDc4TDY5Ljk5MjYgNTcuMTE0NUg3MS41OTY5SDczLjIwMTJMNzQuODg2NSA2My4yNjE5Qzc1LjgxMzQgNjYuNjQzIDc2LjY4MzIgNjkuNzQxMiA3Ni44MTk0IDcwLjE0N0M3Ni45NTU3IDcwLjU1MjcgNzcuODQxOSA2Ny43MzEgNzguNzg5IDYzLjg3NjZMODAuNTEwOSA1Ni44Njg2SDgyLjQ1MTdDODMuOTUxNyA1Ni44Njg2IDg0LjMzMDMgNTcuMDMxNiA4NC4xMTkgNTcuNTg2QzgzLjk2ODYgNTcuOTgwNiA4Mi42OTU2IDYxLjk2NDEgODEuMjkwMiA2Ni40MzgyTDc4LjczNDkgNzQuNTczMUg3Ni45NjM1SDc1LjE5MjFMNzMuNTgxMiA2OC43OTQ1QzcyLjY5NTIgNjUuNjE2MyA3MS44NTg1IDYyLjY4NDEgNzEuNzIxNyA2Mi4yNzgzQzcxLjQ0NjUgNjEuNDYxNiA3MS41OTExIDYxLjA3MzMgNjkuMTExNCA2OS4yODYzTDY3LjUxNTIgNzQuNTczMUg2NS43MjU0SDYzLjkzNTZMNjEuNDEwNyA2Ni4wNTg0WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTE5Ljk3NDYgMTE5LjI3NEMxOS4wMTU4IDExOC4xMTkgMjAuMzM4MSAxMTYuMTY2IDIxLjU2MDQgMTE2LjkzMkMyMi43NjMyIDExNy42ODYgMjIuMzMwNSAxMTkuNzYxIDIwLjk3MDQgMTE5Ljc2MUMyMC42NDUgMTE5Ljc2MSAyMC4xOTY5IDExOS41NDIgMTkuOTc0NiAxMTkuMjc0WiIgZmlsbD0idXJsKCNwYWludDNfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMjMuODM2IDExMi42M0MyMi4xMTUyIDExMS42ODkgMjEuODAyOSAxMDkuNzk3IDIzLjE0NjUgMTA4LjQ1M0MyNC4wODU3IDEwNy41MTQgMjUuMjEgMTA3LjM5NCAyNi4yNTA3IDEwOC4xMjNDMjcuODU3MSAxMDkuMjQ5IDI3LjQ2NTUgMTEyLjAxMyAyNS42MDc5IDExMi42NkMyNS42MDc5IDExMi42NiAyNC40NzQ1IDExMi45NzMgMjMuODM2IDExMi42M1oiIGZpbGw9InVybCgjcGFpbnQ0X3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTMzLjc4NTUgMTIwLjAzNkMzMy43ODU1IDEyMC4xODggMzMuNzU1NyAxMjAuMzM4IDMzLjY5NzYgMTIwLjQ3OUMzMy42Mzk1IDEyMC42MTkgMzMuNTU0NCAxMjAuNzQ2IDMzLjQ0NzEgMTIwLjg1M0MzMy4zMzk4IDEyMC45NjEgMzMuMjEyNCAxMjEuMDQ2IDMzLjA3MjMgMTIxLjEwNEMzMi45MzIxIDEyMS4xNjIgMzIuNzgxOCAxMjEuMTkyIDMyLjYzMDEgMTIxLjE5MkMzMi40NzgzIDEyMS4xOTIgMzIuMzI4MSAxMjEuMTYyIDMyLjE4NzkgMTIxLjEwNEMzMi4wNDc3IDEyMS4wNDYgMzEuOTIwMyAxMjAuOTYxIDMxLjgxMyAxMjAuODUzQzMxLjcwNTcgMTIwLjc0NiAzMS42MjA2IDEyMC42MTkgMzEuNTYyNiAxMjAuNDc5QzMxLjUwNDUgMTIwLjMzOCAzMS40NzQ2IDEyMC4xODggMzEuNDc0NiAxMjAuMDM2QzMxLjQ3NDYgMTE5LjczIDMxLjU5NjMgMTE5LjQzNiAzMS44MTMgMTE5LjIxOUMzMi4wMjk3IDExOS4wMDMgMzIuMzIzNiAxMTguODgxIDMyLjYzMDEgMTE4Ljg4MUMzMi45MzY1IDExOC44ODEgMzMuMjMwNCAxMTkuMDAzIDMzLjQ0NzEgMTE5LjIxOUMzMy42NjM4IDExOS40MzYgMzMuNzg1NSAxMTkuNzMgMzMuNzg1NSAxMjAuMDM2WiIgZmlsbD0iYmxhY2siLz4KPHBhdGggZD0iTTEyMC43MjEgMTE4LjI3NkMxMjAuNzIxIDExOC42MjMgMTIwLjU2NSAxMTguOTU2IDEyMC4yODcgMTE5LjIwMUMxMjAuMDA5IDExOS40NDYgMTE5LjYzMiAxMTkuNTg0IDExOS4yMzkgMTE5LjU4NEMxMTguODQ2IDExOS41ODQgMTE4LjQ2OSAxMTkuNDQ2IDExOC4xOTEgMTE5LjIwMUMxMTcuOTEzIDExOC45NTYgMTE3Ljc1NyAxMTguNjIzIDExNy43NTcgMTE4LjI3NkMxMTcuNzU3IDExOC4xMDUgMTE3Ljc5NSAxMTcuOTM1IDExNy44NyAxMTcuNzc2QzExNy45NDQgMTE3LjYxNyAxMTguMDUzIDExNy40NzMgMTE4LjE5MSAxMTcuMzUyQzExOC4zMjkgMTE3LjIzIDExOC40OTIgMTE3LjEzNCAxMTguNjcyIDExNy4wNjhDMTE4Ljg1MSAxMTcuMDAzIDExOS4wNDQgMTE2Ljk2OSAxMTkuMjM5IDExNi45NjlDMTE5LjQzMyAxMTYuOTY5IDExOS42MjYgMTE3LjAwMyAxMTkuODA2IDExNy4wNjhDMTE5Ljk4NiAxMTcuMTM0IDEyMC4xNDkgMTE3LjIzIDEyMC4yODcgMTE3LjM1MkMxMjAuNDI0IDExNy40NzMgMTIwLjUzMyAxMTcuNjE3IDEyMC42MDggMTE3Ljc3NkMxMjAuNjgyIDExNy45MzUgMTIwLjcyMSAxMTguMTA1IDEyMC43MjEgMTE4LjI3NloiIGZpbGw9IiM1OTY4NkYiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82NzlfMTkxMCIgeDE9IjExIiB5MT0iLTQuNzc3NjhlLTA3IiB4Mj0iMTY0LjUiIHkyPSIxNTAuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNjRCN0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzAwNURBRCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50MV9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMTI0LjkwNCAxMjguNjM2KSByb3RhdGUoLTAuMDQ4NTUyKSBzY2FsZSgzNi4yODAyIDMxLjQwMzIpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50Ml9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMzkuMzUxMiAxMjEuNDYpIHJvdGF0ZSgtMC4wNTczKSBzY2FsZSg1MC4yMjc2IDUxLjMwOTEpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50M19yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjAuOTU0OCAxMTguMjYpIHNjYWxlKDEuMzA5MjkgMS41MDEyNykiPgo8c3RvcCBzdG9wLWNvbG9yPSIjODhBM0QwIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzVEODNCRiIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50NF9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjQuNzg0MyAxMTAuMjIxKSBzY2FsZSgyLjQ3Mjc3IDIuNTY5NTQpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzg4QTNEMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM1RDgzQkYiLz4KPC9yYWRpYWxHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"], ["spec", "lifecycle"], ["spec", "lifecycle", "enabled"], ["spec", "lifecycle", "storageClass"]] secrets: exclude: [] include: [] diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index 3436e5f0f..54a9c0338 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -61,6 +61,12 @@ update: patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch + # Backport of seaweedfs/seaweedfs#11157, released in 4.46: drop the patch + # once the vendored chart carries [grpc.s3]. --forward makes that point + # fail here, where BSD patch would otherwise reverse it and remove the + # section without an error. + patch --forward --no-backup-if-mismatch -p4 < patches/s3-grpc-tls.patch + patch --forward --no-backup-if-mismatch -p4 < patches/storage-class-string.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml $(MAKE) update-image diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml index ad112981f..1ce6f3cab 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml @@ -330,7 +330,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.admin.data.storageClass }} + {{- with .Values.admin.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.admin.data.size }} @@ -344,7 +346,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.admin.logs.storageClass }} + {{- with .Values.admin.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.admin.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml index 4a402f318..f5329a24e 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml @@ -19,7 +19,7 @@ spec: accessModes: {{- toYaml (.Values.allInOne.data.accessModes | default (list "ReadWriteOnce")) | nindent 4 }} {{- if .Values.allInOne.data.storageClass }} - storageClassName: {{ .Values.allInOne.data.storageClass }} + storageClassName: {{ .Values.allInOne.data.storageClass | quote }} {{- end }} resources: requests: diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml index e505faf1c..ced2d597a 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -451,7 +451,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.filer.data.storageClass }} + {{- with .Values.filer.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.filer.data.size }} @@ -467,7 +469,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.filer.logs.storageClass }} + {{- with .Values.filer.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.filer.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml index 914f0dfd1..5df83e687 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml @@ -336,7 +336,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.master.data.storageClass }} + {{- with .Values.master.data.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.master.data.size }} @@ -352,7 +354,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.master.logs.storageClass }} + {{- with .Values.master.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.master.logs.size }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml index a067e1de9..17b82cca0 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml @@ -97,6 +97,19 @@ data: cert = "/usr/local/share/ca-certificates/worker/tls.crt" key = "/usr/local/share/ca-certificates/worker/tls.key" + {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000). + Without this section, LoadServerTLS("grpc.s3") returns nil and the S3 + server serves its internal gRPC (IAM cache propagation, s3 lifecycle + delete RPCs) as plaintext, while peers dial it with mTLS credentials: + "tls: first record does not look like a TLS handshake". + Always uses the internal CA-signed client certificate (already mounted + on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that + secret is for the public HTTPS listener and may be issued by a public + CA which internal gRPC peers (trusting only grpc.ca) would reject. */}} + [grpc.s3] + cert = "/usr/local/share/ca-certificates/client/tls.crt" + key = "/usr/local/share/ca-certificates/client/tls.key" + # use this for any place needs a grpc client # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" [grpc.client] diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml index 04011d1d6..88b13822d 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml @@ -404,7 +404,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $dir.storageClass }} + {{- with $dir.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $dir.size }} @@ -422,7 +424,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $volume.idx.storageClass }} + {{- with $volume.idx.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $volume.idx.size }} @@ -438,7 +442,9 @@ spec: {{- end }} spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ $volume.logs.storageClass }} + {{- with $volume.logs.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ $volume.logs.size }} diff --git a/packages/system/seaweedfs/patches/s3-grpc-tls.patch b/packages/system/seaweedfs/patches/s3-grpc-tls.patch new file mode 100644 index 000000000..d4d45fbd0 --- /dev/null +++ b/packages/system/seaweedfs/patches/s3-grpc-tls.patch @@ -0,0 +1,24 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +index a067e1de9..17b82cca0 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml +@@ -97,6 +97,19 @@ data: + cert = "/usr/local/share/ca-certificates/worker/tls.crt" + key = "/usr/local/share/ca-certificates/worker/tls.key" + ++ {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000). ++ Without this section, LoadServerTLS("grpc.s3") returns nil and the S3 ++ server serves its internal gRPC (IAM cache propagation, s3 lifecycle ++ delete RPCs) as plaintext, while peers dial it with mTLS credentials: ++ "tls: first record does not look like a TLS handshake". ++ Always uses the internal CA-signed client certificate (already mounted ++ on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that ++ secret is for the public HTTPS listener and may be issued by a public ++ CA which internal gRPC peers (trusting only grpc.ca) would reject. */}} ++ [grpc.s3] ++ cert = "/usr/local/share/ca-certificates/client/tls.crt" ++ key = "/usr/local/share/ca-certificates/client/tls.key" ++ + # use this for any place needs a grpc client + # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" + [grpc.client] diff --git a/packages/system/seaweedfs/patches/storage-class-string.patch b/packages/system/seaweedfs/patches/storage-class-string.patch new file mode 100644 index 000000000..0c4562105 --- /dev/null +++ b/packages/system/seaweedfs/patches/storage-class-string.patch @@ -0,0 +1,128 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +index ad112981f..1ce6f3cab 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/admin/admin-statefulset.yaml +@@ -330,7 +330,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.admin.data.storageClass }} ++ {{- with .Values.admin.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.admin.data.size }} +@@ -344,7 +346,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.admin.logs.storageClass }} ++ {{- with .Values.admin.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.admin.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +index 4a402f318..f5329a24e 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml +@@ -19,7 +19,7 @@ spec: + accessModes: + {{- toYaml (.Values.allInOne.data.accessModes | default (list "ReadWriteOnce")) | nindent 4 }} + {{- if .Values.allInOne.data.storageClass }} +- storageClassName: {{ .Values.allInOne.data.storageClass }} ++ storageClassName: {{ .Values.allInOne.data.storageClass | quote }} + {{- end }} + resources: + requests: +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +index e505faf1c..ced2d597a 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/filer/filer-statefulset.yaml +@@ -451,7 +451,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.filer.data.storageClass }} ++ {{- with .Values.filer.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.filer.data.size }} +@@ -467,7 +469,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.filer.logs.storageClass }} ++ {{- with .Values.filer.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.filer.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +index 914f0dfd1..5df83e687 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/master/master-statefulset.yaml +@@ -336,7 +336,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.master.data.storageClass }} ++ {{- with .Values.master.data.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.master.data.size }} +@@ -352,7 +354,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ .Values.master.logs.storageClass }} ++ {{- with .Values.master.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ .Values.master.logs.size }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +index 04011d1d6..88b13822d 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/volume/volume-statefulset.yaml +@@ -404,7 +404,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $dir.storageClass }} ++ {{- with $dir.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $dir.size }} +@@ -422,7 +424,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $volume.idx.storageClass }} ++ {{- with $volume.idx.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $volume.idx.size }} +@@ -438,7 +442,9 @@ spec: + {{- end }} + spec: + accessModes: [ "ReadWriteOnce" ] +- storageClassName: {{ $volume.logs.storageClass }} ++ {{- with $volume.logs.storageClass }} ++ storageClassName: {{ . | quote }} ++ {{- end }} + resources: + requests: + storage: {{ $volume.logs.size }} diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml index 7840d200d..d43aa176e 100644 --- a/packages/system/seaweedfs/tests/cosi_image_test.yaml +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -7,7 +7,7 @@ suite: seaweedfs COSI driver image pin # BucketAccess and ignores the accessPolicy parameter. The chart ships a # -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on # v0.1.2 a credential meant to be read-only is silently granted read-write. -# The package values.yaml overrides cosi.image to v0.3.1, which honours +# The package values.yaml overrides cosi.image to v0.4.0, which honours # accessPolicy: readonly. # # This test does not set the image itself; it relies on the package values.yaml @@ -25,7 +25,7 @@ release: namespace: cozy-seaweedfs tests: - - it: pins the cosi-driver container to v0.3.1 (read-only access enforcement) + - it: pins the cosi-driver container to v0.4.0 (read-only access enforcement) template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: seaweedfs.cosi.enabled: true @@ -35,4 +35,4 @@ tests: value: seaweedfs-cosi-driver - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml new file mode 100644 index 000000000..7f3cec8ff --- /dev/null +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -0,0 +1,175 @@ +suite: seaweedfs runs the S3 lifecycle worker + +# SeaweedFS stores a bucket lifecycle configuration on PUT and acts on it only +# from the s3_lifecycle job of the plugin worker, which the admin server +# schedules (weed/worker/tasks/s3_lifecycle/handler.go). Without an admin and a +# worker in the release, a rule is accepted and nothing ever expires. + +templates: + - charts/seaweedfs/templates/admin/admin-statefulset.yaml + - charts/seaweedfs/templates/worker/worker-deployment.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: runs the admin server that schedules lifecycle jobs + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: &enabled + seaweedfs: + admin: + enabled: true + secret: + existingSecret: seaweedfs-admin-credentials + worker: + enabled: true + asserts: + - isKind: + of: StatefulSet + - equal: + path: metadata.name + value: seaweedfs-admin + + # The admin keeps its job state and any change an operator makes in its UI + # in -dataDir; on an emptyDir a restart would silently undo them. + - it: keeps the admin data directory on a persistent volume + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-dataDir=/data' + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: admin-data + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 1Gi + + - it: runs the worker against the admin of the same release + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - isKind: + of: Deployment + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + + # The worker executes every job type it registers for. Lifecycle is the only + # one this release asks for: vacuum, balance and erasure coding move data and + # are not something enabling lifecycle should start doing as a side effect. + - it: registers the worker for the s3_lifecycle job only + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-jobType=s3_lifecycle ' + + - it: runs no Lance worker container, since the Lance catalog is off + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + + - it: gives the admin and the worker explicit resources + set: *enabled + asserts: + - exists: + path: spec.template.spec.containers[0].resources.requests.memory + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - exists: + path: spec.template.spec.containers[0].resources.limits.memory + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 64Mi + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # The admin serves an unauthenticated UI unless a password is set, and a + # tenant namespace admits ingress from the whole cluster. The credentials come + # from a Secret the parent release keeps; the values here only wire them in. + - it: reads the admin credentials from the named Secret + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: WEED_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: seaweedfs-admin-credentials + key: adminPassword + - contains: + path: spec.template.spec.containers[0].env + content: + name: WEED_ADMIN_USER + valueFrom: + secretKeyRef: + name: seaweedfs-admin-credentials + key: adminUser + + - it: renders neither the admin nor the worker unless the parent turns them on + asserts: + - hasDocuments: + count: 0 + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - hasDocuments: + count: 0 + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # Every other component here logs to stderr; the upstream default for these + # two writes to a file in an emptyDir that kubectl logs never shows. + - it: logs the admin and the worker to stderr + set: *enabled + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-logtostderr=true' + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '-logtostderr=true' + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + + # One worker replica rolls out by surging a second pod. The subchart's hard + # hostname anti-affinity keeps that pod off the only node of a single-node + # cluster, so every later change to the worker template would hang the + # release on its rollout. + - it: lets the worker roll out on a single node + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - notExists: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution + + # The parent hands the class on as a string, and a class name may be all + # digits; emitted unquoted it turns back into a number the claim rejects. + - it: keeps a numeric admin storage class name a string + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs: + admin: + enabled: true + secret: + existingSecret: seaweedfs-admin-credentials + data: + storageClass: "123" + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + + # An empty storageClassName disables dynamic provisioning; with no class + # named, the claim must leave the field out so the cluster default applies. + - it: leaves the admin claim on the cluster default when no class is named + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName diff --git a/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml new file mode 100644 index 000000000..afac836ed --- /dev/null +++ b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml @@ -0,0 +1,32 @@ +suite: seaweedfs s3 gRPC listener speaks mTLS + +templates: + - charts/seaweedfs/templates/shared/security-configmap.yaml + - charts/seaweedfs/templates/s3/s3-deployment.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + # weed s3 serves its internal gRPC (the LifecycleDelete RPC among it) with + # the [grpc.s3] identity and as plaintext without one, while the worker dials + # it with [grpc.worker] mTLS credentials, so every delete failed the TLS + # handshake. patches/s3-grpc-tls.patch backports the section from upstream. + - it: gives the s3 gRPC server a TLS identity signed by the internal CA + template: charts/seaweedfs/templates/shared/security-configmap.yaml + asserts: + - matchRegex: + path: data["security.toml"] + pattern: '\[grpc\.s3\]\s+cert = "/usr/local/share/ca-certificates/client/tls\.crt"\s+key = "/usr/local/share/ca-certificates/client/tls\.key"' + + # security.toml reaches the s3 pods through a subPath mount, which a running + # container never re-reads, so an upgrade that only changes the ConfigMap + # leaves s3 serving plaintext gRPC until something restarts it. The pod + # template has to change in the same release. + - it: rolls the s3 pods so they load the new section + template: charts/seaweedfs/templates/s3/s3-deployment.yaml + asserts: + - equal: + path: spec.template.metadata.annotations["seaweedfs.cozystack.io/grpc-s3-tls"] + value: "true" diff --git a/packages/system/seaweedfs/tests/storage_class_string_test.yaml b/packages/system/seaweedfs/tests/storage_class_string_test.yaml new file mode 100644 index 000000000..038cf1fed --- /dev/null +++ b/packages/system/seaweedfs/tests/storage_class_string_test.yaml @@ -0,0 +1,123 @@ +suite: seaweedfs claims keep the storage class a string + +# A StorageClass name may be all digits. Emitted unquoted it renders as a +# number, which the claim rejects; and an empty storageClassName disables +# dynamic provisioning, so with no class named the field has to stay out for +# the cluster default to apply. + +templates: + - charts/seaweedfs/templates/master/master-statefulset.yaml + - charts/seaweedfs/templates/filer/filer-statefulset.yaml + - charts/seaweedfs/templates/volume/volume-statefulset.yaml + - charts/seaweedfs/templates/admin/admin-statefulset.yaml + - charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml + # filer-statefulset includes it for its checksum annotation. + - charts/seaweedfs/templates/s3/s3-secret.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: keeps numeric master data and logs storage class names strings + template: charts/seaweedfs/templates/master/master-statefulset.yaml + set: + seaweedfs.master.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.master.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[?(@.metadata.name == "data-tenant-root")].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[?(@.metadata.name == "seaweedfs-master-log-volume")].spec.storageClassName + value: "456" + + - it: leaves the master claims on the cluster default when no class is named + template: charts/seaweedfs/templates/master/master-statefulset.yaml + set: + seaweedfs.master.data: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.master.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps numeric filer data and logs storage class names strings + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + set: + seaweedfs.filer.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.filer.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the filer claims on the cluster default when no class is named + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + set: + seaweedfs.filer.data: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.filer.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps numeric volume idx and logs storage class names strings + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + set: + seaweedfs.volume.dataDirs: [] + seaweedfs.volume.idx: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + seaweedfs.volume.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the volume idx and logs claims on the cluster default when no class is named + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + set: + seaweedfs.volume.dataDirs: [] + seaweedfs.volume.idx: {type: persistentVolumeClaim, size: 1Gi} + seaweedfs.volume.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps a numeric admin logs storage class name a string + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs.admin.enabled: true + seaweedfs.admin.logs: {type: persistentVolumeClaim, size: 1Gi, storageClass: "456"} + asserts: + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: "456" + + - it: leaves the admin logs claim on the cluster default when no class is named + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: + seaweedfs.admin.enabled: true + seaweedfs.admin.logs: {type: persistentVolumeClaim, size: 1Gi} + asserts: + - notExists: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: keeps a numeric all-in-one storage class name a string + template: charts/seaweedfs/templates/all-in-one/all-in-one-pvc.yaml + set: + seaweedfs.allInOne.enabled: true + seaweedfs.allInOne.data: {type: persistentVolumeClaim, size: 1Gi, storageClass: "123"} + asserts: + - equal: + path: spec.storageClassName + value: "123" diff --git a/packages/system/seaweedfs/tests/volume_storage_class_test.yaml b/packages/system/seaweedfs/tests/volume_storage_class_test.yaml new file mode 100644 index 000000000..037dc0969 --- /dev/null +++ b/packages/system/seaweedfs/tests/volume_storage_class_test.yaml @@ -0,0 +1,42 @@ +suite: seaweedfs volume claims keep the storage class a string + +templates: + - charts/seaweedfs/templates/volume/volume-statefulset.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + # The parent hands volume.storageClass on as a string, and a class name may + # be all digits; emitted unquoted it turns back into a number the claim + # rejects. + - it: keeps a numeric volume storage class name a string + set: + seaweedfs: + volume: + dataDirs: + - name: data1 + type: persistentVolumeClaim + size: 10Gi + storageClass: "123" + maxVolumes: 0 + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: "123" + + # An empty storageClassName disables dynamic provisioning; with no class + # named, the claim must leave the field out so the cluster default applies. + - it: leaves the volume claim on the cluster default when no class is named + set: + seaweedfs: + volume: + dataDirs: + - name: data1 + type: persistentVolumeClaim + size: 10Gi + maxVolumes: 0 + asserts: + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index bce78c47f..64e2562ba 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -207,6 +207,14 @@ seaweedfs: # weed skip startLanceServer and drops the container port and Service port # with it. lancePort: 0 + # patches/s3-grpc-tls.patch adds [grpc.s3] to security.toml, which the s3 + # pods mount by subPath and so never re-read. This annotation changes the + # pod template in the release that brings the section, so an upgrade + # restarts s3 onto it instead of leaving its gRPC in plaintext. The + # upstream s3 template carries no checksum of that ConfigMap, and adding + # one would be another vendored patch for a change made once. + podAnnotations: + seaweedfs.cozystack.io/grpc-s3-tls: "true" # Keep S3 traffic on same-zone endpoints. Native chart support landed in 4.12 # (replaces the former s3-traffic-distribution.patch); the chart gates the # field on Kubernetes >=1.31 and auto-converts to PreferSameZone on >=1.35. @@ -303,7 +311,10 @@ seaweedfs: # S3 actions for every BucketAccess and ignores the accessPolicy parameter. # v0.3.x honours accessPolicy: readonly, so a -readonly BucketAccessClass # actually issues read-only (Read,List) credentials instead of read-write. - image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1" + # v0.4.0 also writes each grant to the per-identity IAM file: v0.3.1 + # rewrites only the legacy identity.json, which the S3 gateways migrate + # concurrently, so a grant could be lost for good. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" @@ -311,6 +322,62 @@ seaweedfs: region: "" sidecar: image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1" + # A bucket lifecycle rule is stored on PUT and acted on only by the + # s3_lifecycle job of the plugin worker, scheduled by the admin server once a + # day (weed/worker/tasks/s3_lifecycle/handler.go at 4.45). The parent chart + # sends both enabled flags from its lifecycle.enabled, together with the + # admin password below; off here, so a render without the parent never + # starts an admin without one. + admin: + enabled: false + # Without a password weed admin serves its UI unauthenticated, and a tenant + # namespace admits ingress from the whole cluster. The parent chart names + # the Secret it keeps; these are the keys it writes. + secret: + userKey: adminUser + pwKey: adminPassword + # Job state and anything changed in the admin UI live in -dataDir. + data: + type: "persistentVolumeClaim" + size: "1Gi" + logs: + type: "" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + worker: + enabled: false + # Vacuum, balance and erasure coding move data; enabling lifecycle must + # not start them. + jobType: "s3_lifecycle" + # The subchart's hard hostname anti-affinity keeps the surge pod of a + # rollout off the node the running worker holds, so on a single-node + # cluster any change to the worker template never finishes rolling out. + # Only required -> preferred, as for the master above. + affinity: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker + topologyKey: kubernetes.io/hostname + logs: + type: "" + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "1" + memory: "512Mi" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/pkg/apis/apps/defaulting/defaulting.go b/pkg/apis/apps/defaulting/defaulting.go new file mode 100644 index 000000000..f361a7bfd --- /dev/null +++ b/pkg/apis/apps/defaulting/defaulting.go @@ -0,0 +1,201 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package defaulting applies an ApplicationDefinition's structural-schema +// defaults to application values. It is shared by the aggregated apiserver, +// which defaults on the read path, and by anything that has to reason about +// stored values, which carry only the keys a tenant actually wrote. +package defaulting + +import ( + "encoding/json" + "fmt" + "strings" + + internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/runtime" +) + +// BuildSchema parses an OpenAPI-v3 JSON schema string and returns the +// structural-schema form used for defaulting. Returns (nil, nil) when raw +// is empty after trimming. Returns (nil, error) for malformed JSON, +// v1→internal conversion failures, or structural-schema construction +// errors — callers can decide whether to log-and-continue or fail hard. +func BuildSchema(raw string) (*structuralschema.Structural, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + var v1js apiextv1.JSONSchemaProps + if err := json.Unmarshal([]byte(raw), &v1js); err != nil { + return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) + } + scheme := runtime.NewScheme() + _ = internalapiext.AddToScheme(scheme) + _ = apiextv1.AddToScheme(scheme) + var ijs internalapiext.JSONSchemaProps + if err := scheme.Convert(&v1js, &ijs, nil); err != nil { + return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) + } + s, err := structuralschema.NewStructural(&ijs) + if err != nil { + return nil, fmt.Errorf("build structural schema: %w", err) + } + return s, nil +} + +// Apply defaults root in place against s the way the Kubernetes apiserver +// defaults a custom resource. A nil schema leaves root untouched. +func Apply(root *map[string]any, s *structuralschema.Structural) error { + if s == nil { + return nil + } + v := any(*root) + nv, err := applyDefaults(v, s, true) + if err != nil { + return err + } + obj, ok := nv.(map[string]any) + if !ok && nv != nil { + return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) + } + if obj == nil { + obj = map[string]any{} + } + *root = obj + return nil +} + +func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { + if s == nil { + return v, nil + } + + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + default: + // scalar + } + } + + switch effType { + case "object": + mv, isMap := v.(map[string]any) + if !isMap || mv == nil { + if s.Default.Object != nil && !top { + if dm, ok := s.Default.Object.(map[string]any); ok { + mv = cloneMap(dm) + } + } + if mv == nil { + mv = map[string]any{} + } + } + + for name, ps := range s.Properties { + if _, ok := mv[name]; !ok { + if ps.Default.Object != nil { + mv[name] = clone(ps.Default.Object) + } + } + if cur, ok := mv[name]; ok { + cv, err := applyDefaults(cur, &ps, false) + if err != nil { + return nil, err + } + mv[name] = cv + } + } + + if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { + ap := s.AdditionalProperties.Structural + for k, cur := range mv { + if _, isKnown := s.Properties[k]; isKnown { + continue + } + cv, err := applyDefaults(cur, ap, false) + if err != nil { + return nil, err + } + mv[k] = cv + } + } + return mv, nil + + case "array": + sl, isSlice := v.([]any) + if !isSlice || sl == nil { + if s.Default.Object != nil { + if ds, ok := s.Default.Object.([]any); ok { + sl = cloneSlice(ds) + } + } + if sl == nil { + sl = []any{} + } + } + if s.Items != nil { + for i := range sl { + cv, err := applyDefaults(sl[i], s.Items, false) + if err != nil { + return nil, err + } + sl[i] = cv + } + } + return sl, nil + + default: + if v == nil && s.Default.Object != nil { + return clone(s.Default.Object), nil + } + return v, nil + } +} + +func clone(x any) any { + switch t := x.(type) { + case map[string]any: + return cloneMap(t) + case []any: + return cloneSlice(t) + default: + return t + } +} + +func cloneMap(m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = clone(v) + } + return out +} + +func cloneSlice(s []any) []any { + out := make([]any, len(s)) + for i := range s { + out[i] = clone(s[i]) + } + return out +} diff --git a/pkg/apis/apps/defaulting/defaulting_test.go b/pkg/apis/apps/defaulting/defaulting_test.go new file mode 100644 index 000000000..0ce580e56 --- /dev/null +++ b/pkg/apis/apps/defaulting/defaulting_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaulting + +import "testing" + +// A nil map[string]any in an interface is not == nil, so it has to be +// recognised as empty explicitly, at the root and below it, or writing the +// first default into it panics. +func TestApplyDefaultsTypedNilMaps(t *testing.T) { + s, err := BuildSchema(`{"type":"object","properties":{ + "replicas":{"type":"integer","default":2}, + "roles":{"type":"array","default":["worker"],"items":{"type":"string"}}, + "resources":{"type":"object","default":{},"properties":{"preset":{"type":"string","default":"t1.micro"}}}}}`) + if err != nil { + t.Fatal(err) + } + + var root map[string]any + if err := Apply(&root, s); err != nil { + t.Fatal(err) + } + if root["replicas"] != int64(2) { + t.Errorf("root: replicas = %#v, want 2", root["replicas"]) + } + + var nested map[string]any + var roles []any + root = map[string]any{"resources": nested, "roles": roles} + if err := Apply(&root, s); err != nil { + t.Fatal(err) + } + resources, _ := root["resources"].(map[string]any) + if resources["preset"] != "t1.micro" { + t.Errorf("nested: resources = %#v, want the preset default", root["resources"]) + } + if got, _ := root["roles"].([]any); len(got) != 1 || got[0] != "worker" { + t.Errorf("nested: roles = %#v, want the array default", root["roles"]) + } +} diff --git a/pkg/apis/sdn/DESIGN.md b/pkg/apis/sdn/DESIGN.md index e7fa62f95..017da9d9a 100644 --- a/pkg/apis/sdn/DESIGN.md +++ b/pkg/apis/sdn/DESIGN.md @@ -130,7 +130,21 @@ spec: The securitygroup-controller stamps `securitygroup.sdn.cozystack.io/db: ""` onto the pods of the attached `Postgres/db` application, which is what makes the policy's `endpointSelector` select them. -A **membership-only SecurityGroup** — one with `attachments` but no `ingress`/`egress` rules — still projects to a backing CiliumNetworkPolicy, because the controller reconciles the policy (not the SecurityGroup) and the attachments live on it. The CiliumNetworkPolicy CRD requires at least one rule section to be present (its `spec` `anyOf` is `ingress | ingressDeny | egress | egressDeny`), so the projection always emits the `ingress` section, an empty list `ingress: []` when the group has no ingress rules. An empty list is the CRD's documented no-op ("if omitted or empty, this rule does not apply at ingress") and `enableDefaultDeny` defaults to `false` for a direction with no rules, so the policy is schema-valid yet datapath-inert: it neither allows nor denies anything, consistent with §7. The empty section is hidden from the SecurityGroup view on read, so a membership-only group round-trips with no `ingress`/`egress`. +A **membership-only SecurityGroup** — one with `attachments` but no `ingress`/`egress` rules — still projects to a backing CiliumNetworkPolicy, because the controller reconciles the policy (not the SecurityGroup) and the attachments live on it. Cilium rejects a policy none of whose rule sections has an entry: the CRD `anyOf` (`ingress | ingressDeny | egress | egressDeny`) is satisfied by an empty list, but the agent's `api.Rule.Sanitize` counts entries and marks such a policy `Valid=False`. The projection therefore writes one empty ingress rule, which allows nothing, together with `enableDefaultDeny: {ingress: false, egress: false}`, so that the rule does not put the member pods into default-deny either: + +```yaml +spec: + endpointSelector: + matchLabels: + securitygroup.sdn.cozystack.io/db: "" + ingress: + - {} + enableDefaultDeny: + ingress: false + egress: false +``` + +The policy is valid yet datapath-inert: it neither allows nor denies anything, consistent with §7. The override depends on the Cilium agent's `enable-non-default-deny-policies` option, which defaults to `true`; with it off, Cilium forces default deny on every policy. The placeholder is hidden from the SecurityGroup view on read, so a membership-only group round-trips with no `ingress`/`egress`. `enableDefaultDeny` is what tells it apart from a group whose single ingress rule is genuinely empty, which projects without the override and so keeps Cilium's ingress default deny. ## 6. RBAC diff --git a/pkg/lineage/capk_test.go b/pkg/lineage/capk_test.go new file mode 100644 index 000000000..0c17300d2 --- /dev/null +++ b/pkg/lineage/capk_test.go @@ -0,0 +1,129 @@ +package lineage + +import ( + "context" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +const capkNamespace = "tenant-demo" + +// kubernetesAppMapper maps the cluster's HelmRelease to its Kubernetes +// application, as the webhook's ApplicationDefinition mapper does. +type kubernetesAppMapper struct{} + +func (kubernetesAppMapper) Map(*helmv2.HelmRelease) (string, string, string, error) { + return "apps.cozystack.io/v1alpha1", "Kubernetes", "kubernetes-", nil +} + +func capkObject(apiVersion, kind, name string, labels map[string]string, owner *unstructured.Unstructured) *unstructured.Unstructured { + o := &unstructured.Unstructured{} + o.SetAPIVersion(apiVersion) + o.SetKind(kind) + o.SetNamespace(capkNamespace) + o.SetName(name) + o.SetLabels(labels) + if owner != nil { + o.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: owner.GetAPIVersion(), Kind: owner.GetKind(), Name: owner.GetName()}}) + } + return o +} + +// capkNodeChain builds the objects behind the virt-launcher pod of a +// Kubernetes worker node, as observed on a live cluster: the pod is owned by +// its VMI and the VMI by its VM, but CAPK leaves the VM without an +// ownerReference and points at its KubevirtMachine through labels only. From +// the KubevirtMachine up, ownerReferences run to the CAPI Cluster, which the +// cluster's HelmRelease labels. +func capkNodeChain(vmLabels map[string]string) (*unstructured.Unstructured, []runtime.Object, meta.RESTMapper) { + app := capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "cluster-a", nil, nil) + hr := capkObject(HRAPIVersion, HRKind, "kubernetes-cluster-a", nil, nil) + cluster := capkObject("cluster.x-k8s.io/v1beta1", "Cluster", "kubernetes-cluster-a", map[string]string{HRLabel: "kubernetes-cluster-a"}, nil) + md := capkObject("cluster.x-k8s.io/v1beta1", "MachineDeployment", "kubernetes-cluster-a-md0", nil, cluster) + ms := capkObject("cluster.x-k8s.io/v1beta1", "MachineSet", "kubernetes-cluster-a-md0-x", nil, md) + machine := capkObject("cluster.x-k8s.io/v1beta1", "Machine", "kubernetes-cluster-a-md0-x-y", nil, ms) + kvm := capkObject(KubevirtMachineAPIVersion, KubevirtMachineKind, "kubernetes-cluster-a-md0-x-y", nil, machine) + vm := capkObject(VMAPIVersion, VMKind, "kubernetes-cluster-a-md0-x-y", vmLabels, nil) + vmi := capkObject("kubevirt.io/v1", "VirtualMachineInstance", "kubernetes-cluster-a-md0-x-y", nil, vm) + pod := capkObject("v1", "Pod", "virt-launcher-kubernetes-cluster-a-md0-x-y-abcde", nil, vmi) + + objs := []runtime.Object{app, hr, cluster, md, ms, machine, kvm, vm, vmi} + mapper := meta.NewDefaultRESTMapper(nil) + for _, o := range objs { + mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace) + } + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace) + return pod, objs, mapper +} + +func walkNodePod(t *testing.T, vmLabels map[string]string) []ObjectID { + t.Helper() + pod, objs, mapper := capkNodeChain(vmLabels) + dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...) + return WalkOwnershipGraph(context.Background(), dyn, mapper, kubernetesAppMapper{}, pod) +} + +func TestWalkResolvesCAPKNodeVMToKubernetesApp(t *testing.T) { + got := walkNodePod(t, map[string]string{ + KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y", + KubevirtMachineNamespaceLabel: capkNamespace, + }) + want := ObjectID{APIVersion: "apps.cozystack.io/v1alpha1", Kind: "Kubernetes", Namespace: capkNamespace, Name: "cluster-a"} + if len(got) != 1 || got[0] != want { + t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want [%+v]", got, want) + } +} + +func TestWalkFollowsCAPKLabelsOnlyFromKubeVirtVMs(t *testing.T) { + // The CAPK labels name a valid KubevirtMachine in the same namespace in + // every case: only the kind and API group of the labelled object differ, so + // each case fails solely on the guard that restricts the bridge to + // kubevirt.io/v1 VirtualMachines. + capkLabels := map[string]string{ + KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y", + KubevirtMachineNamespaceLabel: capkNamespace, + } + for _, tc := range []struct { + name, apiVersion, kind string + }{ + {name: "other kind in the kubevirt.io group", apiVersion: VMAPIVersion, kind: "Cluster"}, + {name: "VirtualMachineInstance", apiVersion: VMAPIVersion, kind: "VirtualMachineInstance"}, + {name: "VirtualMachine of another group", apiVersion: "example.com/v1", kind: VMKind}, + {name: "VirtualMachine of another kubevirt.io version", apiVersion: "kubevirt.io/v1alpha3", kind: VMKind}, + } { + t.Run(tc.name, func(t *testing.T) { + _, objs, mapper := capkNodeChain(nil) + impostor := capkObject(tc.apiVersion, tc.kind, "impostor", capkLabels, nil) + dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...) + if got := WalkOwnershipGraph(context.Background(), dyn, mapper, kubernetesAppMapper{}, impostor); len(got) != 0 { + t.Fatalf("WalkOwnershipGraph(%s %s) = %+v, want no ancestors", tc.apiVersion, tc.kind, got) + } + }) + } +} + +func TestWalkIgnoresKubevirtMachineOfAnotherNamespace(t *testing.T) { + // A VM pointing at a KubevirtMachine outside its own namespace (an external + // infrastructure cluster, or a forged label) must not be resolved through it. + got := walkNodePod(t, map[string]string{ + KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y", + KubevirtMachineNamespaceLabel: "tenant-other", + }) + if len(got) != 0 { + t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want no ancestors", got) + } +} + +func TestWalkStopsAtUnlabelledVM(t *testing.T) { + // A standalone VM with no CAPK labels and no ownerReference has no ancestors. + if got := walkNodePod(t, nil); len(got) != 0 { + t.Fatalf("WalkOwnershipGraph(pod of standalone VM) = %+v, want no ancestors", got) + } +} diff --git a/pkg/lineage/lineage.go b/pkg/lineage/lineage.go index e1f050052..1546df647 100644 --- a/pkg/lineage/lineage.go +++ b/pkg/lineage/lineage.go @@ -20,6 +20,16 @@ const ( HRAPIVersion = "helm.toolkit.fluxcd.io/v2" HRKind = "HelmRelease" HRLabel = "helm.toolkit.fluxcd.io/name" + + // KubeVirt VirtualMachines created by the Cluster API KubeVirt provider + // (CAPK) for cluster nodes carry no ownerReference: CAPK links them to + // their KubevirtMachine through these labels only. + VMAPIVersion = "kubevirt.io/v1" + VMKind = "VirtualMachine" + KubevirtMachineAPIVersion = "infrastructure.cluster.x-k8s.io/v1alpha1" + KubevirtMachineKind = "KubevirtMachine" + KubevirtMachineNameLabel = "capk.cluster.x-k8s.io/kubevirt-machine-name" + KubevirtMachineNamespaceLabel = "capk.cluster.x-k8s.io/kubevirt-machine-namespace" ) // AppMapper maps HelmRelease to application metadata. @@ -91,6 +101,18 @@ func WalkOwnershipGraph( return } + // A node VM of a Kubernetes cluster has no ownerReference, so without this + // step the walk would stop at it and never reach the cluster's application. + if name, ok := kubevirtMachineName(obj); ok { + ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, KubevirtMachineAPIVersion, KubevirtMachineKind, obj.GetNamespace(), name) + if err != nil { + l.Error(err, "couldn't get KubevirtMachine of VirtualMachine", "namespace", obj.GetNamespace(), "virtualMachine", obj.GetName(), "kubevirtMachine", name) + return + } + out = append(out, WalkOwnershipGraph(ctx, client, mapper, appMapper, ownerObj, state)...) + return + } + // I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics", // like an early return from a function, but if all checks succeed, I don't want to do the rest // of the function, so it's a `for { if err { break } if othererr { break } if allgood { return } @@ -138,6 +160,22 @@ func WalkOwnershipGraph( return } +// kubevirtMachineName returns the KubevirtMachine a CAPK-created +// VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own +// namespace: CAPK can also run against an external infrastructure cluster, +// where the KubevirtMachine lives elsewhere and is not ours to resolve. +func kubevirtMachineName(obj *unstructured.Unstructured) (string, bool) { + if obj.GetAPIVersion() != VMAPIVersion || obj.GetKind() != VMKind { + return "", false + } + labels := obj.GetLabels() + name := labels[KubevirtMachineNameLabel] + if name == "" || labels[KubevirtMachineNamespaceLabel] != obj.GetNamespace() { + return "", false + } + return name, true +} + func parseWalkMemory(memory []interface{}) (*walkState, error) { switch len(memory) { case 0: diff --git a/pkg/registry/apps/application/quota.go b/pkg/registry/apps/application/quota.go index eb6f20622..5376139c0 100644 --- a/pkg/registry/apps/application/quota.go +++ b/pkg/registry/apps/application/quota.go @@ -233,7 +233,7 @@ func (r *REST) validateTenantResourceQuotas(ctx context.Context, app *appsv1alph } parentLimit := parentQuota[res] allocated := siblings[res] // zero value when no sibling bounds this resource - used := poolUsed[renderedLimitKey(res)] + used := poolUsed[usageKey(res)] remaining := parentLimit.DeepCopy() remaining.Sub(allocated) remaining.Sub(used) @@ -285,25 +285,39 @@ var rawQuotaKeys = map[string]struct{}{ "resourcequotas": {}, } -// renderedLimitKey maps a shorthand resourceQuotas key (the vocabulary admission -// works in) to the rendered ResourceQuota key that carries its limit-side -// budget, mirroring cozy-lib.resources.flatten. The mapping is allocation-ratio -// independent: ratios only scale the requests.* side that flatten derives from -// the same shorthand value, so the limit-side key always holds the operator's -// declared amount. This lets admission read a tenant's current usage from its -// rendered ResourceQuota.status.used without replicating the chart's ratio math. +// limitCountedKeys are the shorthand keys whose usage the ResourceQuota pod +// evaluator reports under limits.*. For every other resource it counts only +// requests.*, so limits. or limits.hugepages-* never carries +// usage even though the chart renders it. +var limitCountedKeys = map[string]struct{}{ + "cpu": {}, + "memory": {}, + "ephemeral-storage": {}, +} + +// usageKey maps a shorthand resourceQuotas key (the vocabulary admission works +// in) to the rendered ResourceQuota key whose status.used carries its usage. +// cozy-lib.resources.flatten renders a shorthand key both as limits. and as +// requests.; usage is read from whichever of the two the quota controller +// counts. For cpu, memory and ephemeral-storage that is the limit side, which +// holds the operator's declared amount whatever allocation ratio scales the +// request side. Every other resource is counted only on the request side, where +// flatten applies no ratio, so requests. holds the declared amount too. +// This lets admission read a tenant's current usage from its rendered +// ResourceQuota.status.used without replicating the chart's ratio math. // -// cpu, memory, ephemeral-storage, custom resources -> "limits." +// cpu, memory, ephemeral-storage -> "limits." // storage -> "requests.storage" (no hard limits.storage) // pods, services, services.loadbalancers, ... -> "" (verbatim) -func renderedLimitKey(raw string) string { - if raw == "storage" { - return "requests.storage" - } +// extended resources, hugepages-, ... -> "requests." +func usageKey(raw string) string { if _, ok := rawQuotaKeys[raw]; ok { return raw } - return "limits." + raw + if _, ok := limitCountedKeys[raw]; ok { + return "limits." + raw + } + return "requests." + raw } // parentNamespaceOf returns the namespace owned by the parent of the tenant that @@ -341,7 +355,7 @@ func (r *REST) poolRootOf(ns string, bounded map[string]bool) string { // created/updated). Usage is read from each member's ResourceQuota.status.used, // taking a per-resource max within a namespace to avoid double-counting the // chart-rendered and controller-allocated quotas, and is keyed in the rendered -// quota key space (see renderedLimitKey). Boundedness is decided by declared +// quota key space (see usageKey). Boundedness is decided by declared // resourceQuotas so the result does not depend on whether Flux has rendered a // freshly-created tenant's quota yet. func (r *REST) parentPoolUsage(ctx context.Context, parentNS, excludeNS string) (map[string]resource.Quantity, error) { diff --git a/pkg/registry/apps/application/quota_test.go b/pkg/registry/apps/application/quota_test.go index f6e5d3054..3ea2b7a2b 100644 --- a/pkg/registry/apps/application/quota_test.go +++ b/pkg/registry/apps/application/quota_test.go @@ -136,19 +136,20 @@ func usageQuota(t *testing.T, namespace, name string, used map[string]string) *c } } -func TestRenderedLimitKey(t *testing.T) { +func TestUsageKey(t *testing.T) { cases := map[string]string{ "cpu": "limits.cpu", "memory": "limits.memory", "ephemeral-storage": "limits.ephemeral-storage", - "devices.com/nvidia": "limits.devices.com/nvidia", + "devices.com/nvidia": "requests.devices.com/nvidia", + "hugepages-2Mi": "requests.hugepages-2Mi", "storage": "requests.storage", "pods": "pods", "services.loadbalancers": "services.loadbalancers", } for raw, want := range cases { - if got := renderedLimitKey(raw); got != want { - t.Errorf("renderedLimitKey(%q) = %q, want %q", raw, got, want) + if got := usageKey(raw); got != want { + t.Errorf("usageKey(%q) = %q, want %q", raw, got, want) } } } @@ -339,6 +340,26 @@ func TestValidateTenantResourceQuotas_PoolUsage(t *testing.T) { } }) + t.Run("parent usage of an extended resource shrinks remaining", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"}) + used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "2"}) + r := newTenantREST(t, parent, used) + + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) == 0 { + t.Fatalf("child example.com/device=1 should be denied when the parent pool already uses 2 of 2") + } + }) + + t.Run("parent usage of an extended resource leaves the rest grantable", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"}) + used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "1"}) + r := newTenantREST(t, parent, used) + + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) > 0 { + t.Fatalf("child example.com/device=1 should be allowed when the parent pool uses 1 of 2, got: %v", errs) + } + }) + // No rendered usage yet (freshly-created parent, Flux has not rendered the // quota): usage degrades to zero and behaviour matches the pre-usage gate. t.Run("absent usage degrades to budget-only check", func(t *testing.T) { diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 5f618b4e3..aa0e24a62 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fluxshard "github.com/cozystack/cozystack/internal/fluxshardoperator" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" "github.com/cozystack/cozystack/pkg/apis/apps/presets" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/apis/apps/validation" @@ -57,7 +58,6 @@ import ( "github.com/cozystack/cozystack/pkg/registry" fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields" "github.com/cozystack/cozystack/pkg/registry/sorting" - internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -110,26 +110,7 @@ type REST struct { // errors — callers can decide whether to log-and-continue (NewREST does) // or fail hard (tests do). func buildSpecSchema(raw string) (*structuralschema.Structural, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - var v1js apiextv1.JSONSchemaProps - if err := json.Unmarshal([]byte(raw), &v1js); err != nil { - return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) - } - scheme := runtime.NewScheme() - _ = internalapiext.AddToScheme(scheme) - _ = apiextv1.AddToScheme(scheme) - var ijs internalapiext.JSONSchemaProps - if err := scheme.Convert(&v1js, &ijs, nil); err != nil { - return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) - } - s, err := structuralschema.NewStructural(&ijs) - if err != nil { - return nil, fmt.Errorf("build structural schema: %w", err) - } - return s, nil + return defaulting.BuildSchema(raw) } // NewREST creates a new REST storage for Application with specific configuration diff --git a/pkg/registry/apps/application/rest_defaulting.go b/pkg/registry/apps/application/rest_defaulting.go index 36518af69..4fe7409dd 100644 --- a/pkg/registry/apps/application/rest_defaulting.go +++ b/pkg/registry/apps/application/rest_defaulting.go @@ -18,8 +18,8 @@ package application import ( "encoding/json" - "fmt" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -51,137 +51,5 @@ func (r *REST) applySpecDefaults(app *appsv1alpha1.Application) error { } func defaultLikeKubernetes(root *map[string]any, s *structuralschema.Structural) error { - v := any(*root) - nv, err := applyDefaults(v, s, true) - if err != nil { - return err - } - obj, ok := nv.(map[string]any) - if !ok && nv != nil { - return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) - } - if obj == nil { - obj = map[string]any{} - } - *root = obj - return nil -} - -func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { - if s == nil { - return v, nil - } - - effType := s.Type - if effType == "" { - switch { - case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): - effType = "object" - case s.Items != nil: - effType = "array" - default: - // scalar - } - } - - switch effType { - case "object": - mv, isMap := v.(map[string]any) - if !isMap || v == nil { - if s.Default.Object != nil && !top { - if dm, ok := s.Default.Object.(map[string]any); ok { - mv = cloneMap(dm) - } - } - if mv == nil { - mv = map[string]any{} - } - } - - for name, ps := range s.Properties { - if _, ok := mv[name]; !ok { - if ps.Default.Object != nil { - mv[name] = clone(ps.Default.Object) - } - } - if cur, ok := mv[name]; ok { - cv, err := applyDefaults(cur, &ps, false) - if err != nil { - return nil, err - } - mv[name] = cv - } - } - - if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { - ap := s.AdditionalProperties.Structural - for k, cur := range mv { - if _, isKnown := s.Properties[k]; isKnown { - continue - } - cv, err := applyDefaults(cur, ap, false) - if err != nil { - return nil, err - } - mv[k] = cv - } - } - return mv, nil - - case "array": - sl, isSlice := v.([]any) - if !isSlice || v == nil { - if s.Default.Object != nil { - if ds, ok := s.Default.Object.([]any); ok { - sl = cloneSlice(ds) - } - } - if sl == nil { - sl = []any{} - } - } - if s.Items != nil { - for i := range sl { - cv, err := applyDefaults(sl[i], s.Items, false) - if err != nil { - return nil, err - } - sl[i] = cv - } - } - return sl, nil - - default: - if v == nil && s.Default.Object != nil { - return clone(s.Default.Object), nil - } - return v, nil - } -} - -func clone(x any) any { - switch t := x.(type) { - case map[string]any: - return cloneMap(t) - case []any: - return cloneSlice(t) - default: - return t - } -} - -func cloneMap(m map[string]any) map[string]any { - out := make(map[string]any, len(m)) - for k, v := range m { - out[k] = clone(v) - } - return out -} - -func cloneSlice(s []any) []any { - out := make([]any, len(s)) - for i := range s { - out[i] = clone(s[i]) - } - return out + return defaulting.Apply(root, s) } diff --git a/pkg/registry/sdn/securitygroup/cilium.go b/pkg/registry/sdn/securitygroup/cilium.go index fa0ddb65a..906afcdcd 100644 --- a/pkg/registry/sdn/securitygroup/cilium.go +++ b/pkg/registry/sdn/securitygroup/cilium.go @@ -44,25 +44,24 @@ type CiliumNetworkPolicySpec struct { // it to the SecurityGroup's own membership label. EndpointSelector metav1.LabelSelector `json:"endpointSelector,omitempty"` - // Ingress is the list of allowed inbound traffic rules. It is always - // serialized (an empty list, not omitted) and the projection guarantees it is - // non-nil. The backing CiliumNetworkPolicy CRD requires at least one rule - // section to be present (spec anyOf: ingress|ingressDeny|egress|egressDeny), - // so a rules-less (membership-only) SecurityGroup would otherwise project to a - // selector-only spec the cilium.io/v2 API rejects synchronously. An empty - // ingress list is the CRD's documented no-op ("if omitted or empty, this rule - // does not apply at ingress"), and enableDefaultDeny defaults to false for a - // direction with no rules, so the always-present list keeps the policy - // schema-valid while leaving the member pods' connectivity untouched. The - // non-nil guarantee matters because the CRD has no nullable fields: a nil - // slice without omitempty would marshal to null and be rejected. - Ingress []CiliumIngressRule `json:"ingress"` + // Ingress is the list of allowed inbound traffic rules. + Ingress []CiliumIngressRule `json:"ingress,omitempty"` - // Egress is the list of allowed outbound traffic rules. Unlike Ingress it - // keeps omitempty: ingress is the guaranteed-present carrier that satisfies the - // CRD anyOf in every case, so egress is emitted only when there are egress - // rules. + // Egress is the list of allowed outbound traffic rules. Egress []CiliumEgressRule `json:"egress,omitempty"` + + // EnableDefaultDeny overrides Cilium's per-direction default-deny posture. + // The projection sets it only for a membership-only SecurityGroup; see + // membershipOnlySpec. + EnableDefaultDeny *CiliumDefaultDeny `json:"enableDefaultDeny,omitempty"` +} + +// CiliumDefaultDeny mirrors the cilium.io/v2 enableDefaultDeny block. A nil +// direction leaves Cilium's default in place: deny for a direction that has +// rules, allow otherwise. +type CiliumDefaultDeny struct { + Ingress *bool `json:"ingress,omitempty"` + Egress *bool `json:"egress,omitempty"` } // CiliumIngressRule mirrors a single cilium.io/v2 ingress rule. fromApp/fromSG @@ -170,6 +169,26 @@ func (in *CiliumNetworkPolicySpec) DeepCopyInto(out *CiliumNetworkPolicySpec) { in.Egress[i].DeepCopyInto(&out.Egress[i]) } } + if in.EnableDefaultDeny != nil { + out.EnableDefaultDeny = in.EnableDefaultDeny.DeepCopy() + } +} + +// DeepCopy returns a deep copy of the receiver. +func (in *CiliumDefaultDeny) DeepCopy() *CiliumDefaultDeny { + if in == nil { + return nil + } + out := new(CiliumDefaultDeny) + if in.Ingress != nil { + v := *in.Ingress + out.Ingress = &v + } + if in.Egress != nil { + v := *in.Egress + out.Egress = &v + } + return out } // DeepCopy returns a deep copy of the receiver. diff --git a/pkg/registry/sdn/securitygroup/rest.go b/pkg/registry/sdn/securitygroup/rest.go index a06439a75..d4a8d223d 100644 --- a/pkg/registry/sdn/securitygroup/rest.go +++ b/pkg/registry/sdn/securitygroup/rest.go @@ -143,11 +143,10 @@ func sgFromSelector(sel metav1.LabelSelector) (string, bool) { // projectIngress turns the SecurityGroup ingress rules into Cilium ingress // rules: fromApp peers become lineage-label endpointSelectors, fromSG peers // become membership-label endpointSelectors, and fromCIDR/toPorts carry over. -// It always returns a non-nil slice (empty when there are no rules): the backing -// CiliumNetworkPolicy's ingress section must be present on the wire to satisfy -// the CRD anyOf, and a non-nil empty slice serializes to an empty list rather -// than null (which the CRD, having no nullable fields, would reject). func projectIngress(rules []sdnv1alpha1.IngressRule) []CiliumIngressRule { + if rules == nil { + return nil + } out := make([]CiliumIngressRule, len(rules)) for i := range rules { var eps []metav1.LabelSelector @@ -195,9 +194,8 @@ func projectEgress(rules []sdnv1alpha1.EgressRule) []CiliumEgressRule { // the lineage-label nor the membership-label shape are ignored, so a // hand-edited backing policy degrades gracefully rather than erroring. func reconstructIngress(rules []CiliumIngressRule) []sdnv1alpha1.IngressRule { - // Collapse an empty list to nil so the always-present empty ingress section the - // projection emits for a rules-less group (to satisfy the CRD anyOf) is - // invisible in the SecurityGroup view, keeping the rules-less round-trip exact. + // Collapse an empty list to nil: policies written before membershipOnlySpec + // carried an always-present empty ingress section. if len(rules) == 0 { return nil } @@ -224,8 +222,8 @@ func reconstructIngress(rules []CiliumIngressRule) []sdnv1alpha1.IngressRule { // reconstructEgress is the egress counterpart of reconstructIngress. func reconstructEgress(rules []CiliumEgressRule) []sdnv1alpha1.EgressRule { - // Collapse an empty list to nil for the same reason as reconstructIngress, so a - // SecurityGroup that carries no egress rules round-trips with egress unset. + // Collapse an empty list to nil, so a SecurityGroup that carries no egress + // rules round-trips with egress unset. if len(rules) == 0 { return nil } @@ -409,8 +407,10 @@ func policyToSecurityGroup(np *CiliumNetworkPolicy) *sdnv1alpha1.SecurityGroup { // Attachments live in a storage-owned annotation, not the // endpointSelector (which is the SecurityGroup's own membership label). Attachments: decodeAttachments(np.Annotations[attachmentsAnnotation]), - Ingress: reconstructIngress(spec.Ingress), - Egress: reconstructEgress(spec.Egress), + } + if !isMembershipOnlySpec(spec) { + sg.Spec.Ingress = reconstructIngress(spec.Ingress) + sg.Spec.Egress = reconstructEgress(spec.Egress) } } return sg @@ -485,6 +485,10 @@ func securityGroupToPolicy(sg *sdnv1alpha1.SecurityGroup, cur *CiliumNetworkPoli // label onto the attached applications' pods. Rules project app/SG peers into // endpoint selectors. spec := sg.Spec.DeepCopy() + if len(spec.Ingress) == 0 && len(spec.Egress) == 0 { + out.Spec = membershipOnlySpec(sg.Name) + return &out + } out.Spec = &CiliumNetworkPolicySpec{ EndpointSelector: buildEndpointSelector(sg.Name), Ingress: projectIngress(spec.Ingress), @@ -497,6 +501,41 @@ func securityGroupToPolicy(sg *sdnv1alpha1.SecurityGroup, cur *CiliumNetworkPoli return &out } +// membershipOnlySpec is the backing spec of a SecurityGroup with no rules. The +// policy must still exist, since it carries the attachments the +// securitygroup-controller reconciles, but Cilium rejects a rule with no +// ingress, ingressDeny, egress or egressDeny entry (api.Rule.Sanitize counts +// entries, so an empty list is as good as absent) and marks the policy +// Valid=False. One empty ingress rule satisfies that check and allows nothing, +// and disabling default deny in both directions keeps that rule from putting +// the member pods into default-deny, so the policy changes nothing in the +// datapath. That relies on enable-non-default-deny-policies, which defaults to +// true; with it off Cilium forces default deny on every policy. +func membershipOnlySpec(name string) *CiliumNetworkPolicySpec { + off := false + return &CiliumNetworkPolicySpec{ + EndpointSelector: buildEndpointSelector(name), + Ingress: []CiliumIngressRule{{}}, + EnableDefaultDeny: &CiliumDefaultDeny{Ingress: &off, Egress: &off}, + } +} + +// isMembershipOnlySpec reports whether spec is what membershipOnlySpec writes, +// so the reverse projection shows no rules for it. enableDefaultDeny tells it +// apart from a SecurityGroup whose single ingress rule is genuinely empty, +// which the projection writes without it, keeping ingress default deny. +func isMembershipOnlySpec(spec *CiliumNetworkPolicySpec) bool { + d := spec.EnableDefaultDeny + if d == nil || d.Ingress == nil || *d.Ingress || d.Egress == nil || *d.Egress { + return false + } + if len(spec.Egress) != 0 || len(spec.Ingress) != 1 { + return false + } + in := spec.Ingress[0] + return len(in.FromEndpoints) == 0 && len(in.FromCIDR) == 0 && len(in.ToPorts) == 0 +} + // normalizePortProtocols upper-cases every port protocol in place so the value // written to the backing CiliumNetworkPolicy matches its case-sensitive enum. func normalizePortProtocols(spec *CiliumNetworkPolicySpec) { diff --git a/pkg/registry/sdn/securitygroup/rest_test.go b/pkg/registry/sdn/securitygroup/rest_test.go index 5473dfd56..f3ad68fc3 100644 --- a/pkg/registry/sdn/securitygroup/rest_test.go +++ b/pkg/registry/sdn/securitygroup/rest_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "sort" "strings" @@ -1195,16 +1196,53 @@ func TestCreateAcceptsEmptyAttachments(t *testing.T) { } } +// specWire returns the backing policy's spec as Cilium decodes it: the CRD and +// the agent read the wire form, not the Go struct. +func specWire(t *testing.T, np *CiliumNetworkPolicy) map[string]any { + t.Helper() + raw, err := json.Marshal(np) + if err != nil { + t.Fatalf("marshal backing policy: %v", err) + } + var decoded struct { + Spec map[string]any `json:"spec"` + } + if err := json.Unmarshal(raw, &decoded); err != nil { + t.Fatalf("unmarshal backing policy: %v", err) + } + return decoded.Spec +} + +// ciliumAcceptsRuleSections mirrors the first check of Cilium's +// api.Rule.Sanitize, which marks a policy Valid=False when none of its rule +// sections has an entry. The CRD anyOf only needs one of the keys to be +// present, so it admits an empty list that the agent then rejects. +func ciliumAcceptsRuleSections(spec map[string]any) bool { + for _, k := range []string{"ingress", "ingressDeny", "egress", "egressDeny"} { + if l, _ := spec[k].([]any); len(l) > 0 { + return true + } + } + return false +} + +func getBackingPolicy(t *testing.T, r *REST, name string) *CiliumNetworkPolicy { + t.Helper() + np := &CiliumNetworkPolicy{} + if err := r.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: name}, np); err != nil { + t.Fatalf("backing policy not found: %v", err) + } + return np +} + func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) { r := newTestREST(t) // A membership-only SecurityGroup attaches to an application to stamp the - // membership label but carries no ingress or egress rules. The backing - // CiliumNetworkPolicy CRD still requires at least one rule section to be - // present (spec anyOf: ingress|ingressDeny|egress|egressDeny), so a - // selector-only spec is rejected synchronously by the cilium.io/v2 API. The - // projection must emit an explicit empty ingress list — the CRD's documented - // no-op ("if omitted or empty, this rule does not apply at ingress") — so the - // policy is schema-valid yet leaves the member pods' connectivity untouched. + // membership label but carries no ingress or egress rules. Its backing + // policy must be one Cilium accepts and that changes nothing in the + // datapath: a rule section with an entry that allows nothing, and default + // deny disabled in both directions so that entry does not isolate the + // member pods. createSG(t, r, &sdnv1alpha1.SecurityGroup{ ObjectMeta: metav1.ObjectMeta{Name: "sg-membership", Namespace: testNamespace}, Spec: sdnv1alpha1.SecurityGroupSpec{ @@ -1212,50 +1250,22 @@ func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) { }, }) - np := &CiliumNetworkPolicy{} - if err := r.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: "sg-membership"}, np); err != nil { - t.Fatalf("backing policy not found: %v", err) + spec := specWire(t, getBackingPolicy(t, r, "sg-membership")) + if !ciliumAcceptsRuleSections(spec) { + t.Fatalf("rules-less policy has no rule entry, Cilium marks it Valid=False: %v", spec) + } + if got, want := fmt.Sprint(spec["ingress"]), "[map[]]"; got != want { + t.Fatalf("rules-less policy ingress = %s, want %s (one rule that allows nothing)", got, want) + } + if _, ok := spec["egress"]; ok { + t.Fatalf("rules-less policy must not carry an egress key: %v", spec) + } + if got, want := fmt.Sprint(spec["enableDefaultDeny"]), "map[egress:false ingress:false]"; got != want { + t.Fatalf("rules-less policy enableDefaultDeny = %s, want %s", got, want) } - // The CRD validates the wire form, not the Go struct, so assert on the - // marshaled bytes: spec.ingress must be present (an empty list satisfies the - // anyOf), and spec.egress must be absent (ingress is the guaranteed-present - // carrier; egress is emitted only when there are egress rules). - raw, err := json.Marshal(np) - if err != nil { - t.Fatalf("marshal backing policy: %v", err) - } - var decoded struct { - Spec map[string]json.RawMessage `json:"spec"` - } - if err := json.Unmarshal(raw, &decoded); err != nil { - t.Fatalf("unmarshal backing policy: %v", err) - } - ing, ok := decoded.Spec["ingress"] - if !ok { - t.Fatalf("backing policy spec has no ingress key; the CRD anyOf would reject it: %s", raw) - } - if string(ing) != "[]" { - t.Fatalf("rules-less policy ingress = %s, want []", ing) - } - if _, ok := decoded.Spec["egress"]; ok { - t.Fatalf("rules-less policy must not carry an egress key: %s", raw) - } - - // The same key must survive the unstructured conversion path (which honors - // omitempty independently of json.Marshal), so the empty list cannot be - // silently dropped there either. - u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(np) - if err != nil { - t.Fatalf("to unstructured: %v", err) - } - spec, _ := u["spec"].(map[string]any) - if _, ok := spec["ingress"]; !ok { - t.Fatalf("unstructured backing policy spec has no ingress key: %v", spec) - } - - // On read, the synthetic empty ingress list collapses back to no rules, so a - // rules-less SecurityGroup round-trips with neither ingress nor egress. + // On read, the placeholder rule is hidden, so a rules-less SecurityGroup + // round-trips with neither ingress nor egress. out, err := r.Get(ctxNS(), "sg-membership", &metav1.GetOptions{}) if err != nil { t.Fatalf("Get returned error: %v", err) @@ -1264,6 +1274,43 @@ func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) { if got.Spec.Ingress != nil || got.Spec.Egress != nil { t.Fatalf("rules-less round-trip should carry no ingress/egress, got: %+v", got.Spec) } + + // Adding a rule must drop the default-deny override, or the group's + // rules would allow traffic without isolating the member pods. + got.Spec.Egress = []sdnv1alpha1.EgressRule{{ToCIDR: []string{"10.0.0.0/8"}}} + if _, _, err := r.Update(ctxNS(), "sg-membership", rest.DefaultUpdatedObjectInfo(got), nil, nil, false, &metav1.UpdateOptions{}); err != nil { + t.Fatalf("Update returned error: %v", err) + } + spec = specWire(t, getBackingPolicy(t, r, "sg-membership")) + if _, ok := spec["enableDefaultDeny"]; ok { + t.Fatalf("policy with rules must not override default deny: %v", spec) + } + if _, ok := spec["ingress"]; ok { + t.Fatalf("egress-only policy must not carry an ingress key: %v", spec) + } +} + +func TestEmptyIngressRuleIsNotMembershipOnly(t *testing.T) { + r := newTestREST(t) + // An explicit empty ingress rule projects to the same rule as the + // membership-only placeholder, but it is tenant-authored: it keeps Cilium's + // ingress default deny and must stay visible on read. + createSG(t, r, &sdnv1alpha1.SecurityGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "sg-deny-in", Namespace: testNamespace}, + Spec: sdnv1alpha1.SecurityGroupSpec{Ingress: []sdnv1alpha1.IngressRule{{}}}, + }) + + spec := specWire(t, getBackingPolicy(t, r, "sg-deny-in")) + if _, ok := spec["enableDefaultDeny"]; ok { + t.Fatalf("an empty ingress rule must keep default deny: %v", spec) + } + out, err := r.Get(ctxNS(), "sg-deny-in", &metav1.GetOptions{}) + if err != nil { + t.Fatalf("Get returned error: %v", err) + } + if got := out.(*sdnv1alpha1.SecurityGroup).Spec.Ingress; len(got) != 1 { + t.Fatalf("empty ingress rule should round-trip as one rule, got: %+v", got) + } } func TestCreateRejectsInvalidAttachmentLabelValue(t *testing.T) { diff --git a/pkg/reservation/aggregate.go b/pkg/reservation/aggregate.go new file mode 100644 index 000000000..635c41284 --- /dev/null +++ b/pkg/reservation/aggregate.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import "fmt" + +// Application is one stored application as the aggregator sees it: the +// labelled HelmRelease the aggregated apiserver persisted, reduced to what +// identifies its definition and its undefaulted values. +type Application struct { + Namespace string + Name string + Kind string + Group string + Values []byte +} + +// EvaluatorLookup returns the evaluator of a kind, found by its kind and +// group so that an out-of-tree kind resolves through the same path as an +// in-tree one. +type EvaluatorLookup func(kind, group string) (*Evaluator, error) + +// Failure is an application whose reservation could not be computed. +type Failure struct { + Application Application + Err error +} + +func (f Failure) Error() string { + return fmt.Sprintf("%s %s/%s: %v", f.Application.Kind, f.Application.Namespace, f.Application.Name, f.Err) +} + +// Aggregate sums the reservations of apps per namespace. +// +// An application that cannot be evaluated is returned as a Failure and is +// left out of the sums. The sums are therefore a lower bound whenever a +// failure is returned, and the caller must not treat them as complete: a +// stored application that stops evaluating has to keep its last known +// reservation rather than hand its share of the pool to the next writer. +func Aggregate(apps []Application, lookup EvaluatorLookup, env Environment) (map[string]ResourceList, []Failure) { + perNamespace := map[string]ResourceList{} + var failures []Failure + for _, app := range apps { + ev, err := lookup(app.Kind, app.Group) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + rl, err := ev.EvaluateJSON(app.Values, env) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + sum, ok := perNamespace[app.Namespace] + if !ok { + sum = ResourceList{} + perNamespace[app.Namespace] = sum + } + sum.Add(rl) + } + return perNamespace, failures +} diff --git a/pkg/reservation/environment.go b/pkg/reservation/environment.go new file mode 100644 index 000000000..f6727c867 --- /dev/null +++ b/pkg/reservation/environment.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ErrNotFound is wrapped by an Environment when a named instance type or +// preset does not exist. The evaluator fails closed on it rather than +// reserving nothing. +var ErrNotFound = errors.New("not found") + +// Environment resolves the cluster objects a declaration may reference by +// name. Implementations must be safe for concurrent use and return a list +// the caller may keep: the evaluator does not copy it. +type Environment interface { + Instancetype(name string) (ResourceList, error) + Preset(name string) (ResourceList, error) +} + +// StaticEnvironment is an Environment over fixed tables. +type StaticEnvironment struct { + Instancetypes map[string]ResourceList + Presets map[string]ResourceList +} + +var _ Environment = (*StaticEnvironment)(nil) + +// NewStaticEnvironment classifies every VirtualMachineClusterInstancetype in +// instancetypes and pairs the result with presets. One unclassifiable type +// fails the whole environment, so that a broken catalog is noticed when it is +// loaded rather than when a tenant first orders that type. +func NewStaticEnvironment(instancetypes []map[string]any, presets map[string]ResourceList) (*StaticEnvironment, error) { + env := &StaticEnvironment{ + Instancetypes: make(map[string]ResourceList, len(instancetypes)), + Presets: presets, + } + for _, obj := range instancetypes { + rl, err := ClassifyInstancetype(obj) + if err != nil { + return nil, err + } + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + if name == "" { + return nil, fmt.Errorf("instance type without metadata.name") + } + env.Instancetypes[name] = rl + } + return env, nil +} + +func (e *StaticEnvironment) Instancetype(name string) (ResourceList, error) { + rl, ok := e.Instancetypes[name] + if !ok { + return nil, fmt.Errorf("instance type %q: %w", name, ErrNotFound) + } + return rl, nil +} + +func (e *StaticEnvironment) Preset(name string) (ResourceList, error) { + rl, ok := e.Presets[name] + if !ok { + return nil, fmt.Errorf("resource preset %q: %w", name, ErrNotFound) + } + return rl, nil +} diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go new file mode 100644 index 000000000..6927b0bf2 --- /dev/null +++ b/pkg/reservation/evaluator.go @@ -0,0 +1,453 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + "math" + "regexp" + "slices" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/apiserver/pkg/cel/library" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// ErrNoDeclaration is returned for a kind whose definition declares neither +// its consumption nor an exemption. Such a kind is refused rather than +// treated as free: for an out-of-tree catalog an omitted declaration would +// otherwise be a quota-escalation vector no in-tree test can reach. +var ErrNoDeclaration = errors.New("the application definition declares neither its consumption nor an exemption") + +const ( + valuesVar = "values" + instancetypeFunc = "instanceType" + presetFunc = "preset" + + // costLimit bounds one expression. Declarations are platform-authored, + // but they run on the admission path, so a runaway comprehension must + // fail rather than stall a write. + costLimit = 1_000_000 +) + +var bindingName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// Evaluator computes the reservation of one kind's applications. It is built +// once per declaration and is safe for concurrent use. +type Evaluator struct { + exempt bool + schema *structuralschema.Structural + env *cel.Env + lets []compiled + resources []compiled +} + +type compiled struct { + name string + ast *cel.Ast +} + +// Compile checks decl and prepares it for evaluation. schema is the kind's +// structural OpenAPI schema; stored values are defaulted against it before +// any expression reads them, because the aggregated apiserver persists a +// tenant's spec without its defaults. A nil decl yields ErrNoDeclaration. +func Compile(decl *Declaration, schema *structuralschema.Structural) (*Evaluator, error) { + if decl == nil { + return nil, ErrNoDeclaration + } + if decl.Exempt { + if len(decl.Let) > 0 || len(decl.Resources) > 0 { + return nil, fmt.Errorf("an exempt declaration must not carry let or resources") + } + return &Evaluator{exempt: true}, nil + } + if len(decl.Resources) == 0 { + return nil, ErrNoDeclaration + } + + opts := []cel.EnvOption{ + cel.Variable(valuesVar, cel.DynType), + library.Quantity(), + cel.Function(instancetypeFunc, + cel.Overload("instanceType_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function(presetFunc, + cel.Overload("preset_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function("quantity", + cel.Overload("quantity_int", []*cel.Type{cel.IntType}, apiservercel.QuantityType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + return apiservercel.Quantity{Quantity: resource.NewQuantity(int64(v.(types.Int)), resource.DecimalSI)} + }))), + // The standard _*_ is a singleton dispatching on the left operand's + // traits, so it cannot take a quantity overload; scaling is a method. + cel.Function("mul", + cel.MemberOverload("quantity_mul_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, + cel.BinaryBinding(mulQuantity))), + } + seen := map[string]bool{valuesVar: true} + for _, b := range decl.Let { + if !bindingName.MatchString(b.Name) { + return nil, fmt.Errorf("let %q: not a valid identifier", b.Name) + } + if seen[b.Name] { + return nil, fmt.Errorf("let %q: name already bound", b.Name) + } + seen[b.Name] = true + opts = append(opts, cel.Variable(b.Name, cel.DynType)) + } + env, err := cel.NewEnv(opts...) + if err != nil { + return nil, fmt.Errorf("build CEL environment: %w", err) + } + + ev := &Evaluator{schema: schema, env: env} + for _, b := range decl.Let { + ast, iss := env.Compile(b.Expression) + if iss.Err() != nil { + return nil, fmt.Errorf("let %q: %w", b.Name, iss.Err()) + } + ev.lets = append(ev.lets, compiled{name: b.Name, ast: ast}) + } + for _, key := range slices.Sorted(maps.Keys(decl.Resources)) { + if key == "" { + return nil, fmt.Errorf("resources: empty quota key") + } + ast, iss := env.Compile(decl.Resources[key]) + if iss.Err() != nil { + return nil, fmt.Errorf("resources[%q]: %w", key, iss.Err()) + } + ev.resources = append(ev.resources, compiled{name: key, ast: ast}) + } + return ev, nil +} + +// Exempt reports whether the kind declared that it consumes nothing. +func (e *Evaluator) Exempt() bool { return e.exempt } + +// EvaluateJSON evaluates the reservation of an application from its values +// as stored, a JSON object. +func (e *Evaluator) EvaluateJSON(raw []byte, env Environment) (ResourceList, error) { + values := map[string]any{} + if len(bytes.TrimSpace(raw)) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("decode values: %w", err) + } + m, ok := v.(map[string]any) + if !ok && v != nil { + return nil, fmt.Errorf("decode values: expected an object, got %T", v) + } + if m != nil { + values = m + } + } + return e.Evaluate(values, env) +} + +// Evaluate returns the reservation of an application with the given values. +// The values are defaulted first and are not modified. Anything that cannot +// be resolved — an unknown instance type, a value of the wrong type, a key an +// expression reads without guarding it with has() — is an error, never a +// smaller figure. Keys that evaluate to zero are omitted. +func (e *Evaluator) Evaluate(values map[string]any, env Environment) (ResourceList, error) { + if e.exempt { + return ResourceList{}, nil + } + if env == nil { + return nil, fmt.Errorf("no environment to resolve instance types and presets against") + } + + defaulted := normalizeNumbers(values).(map[string]any) + // Defaulting replaces a value of the wrong type with the schema's + // default, which would turn a malformed spec into a smaller one. + if err := checkTypes(defaulted, e.schema, ""); err != nil { + return nil, err + } + if err := defaulting.Apply(&defaulted, e.schema); err != nil { + return nil, fmt.Errorf("default values: %w", err) + } + + // The environment differs per call, so instanceType and preset are + // declared with a late binding and bound here, per program; cel.Functions + // is deprecated for declaring functions but is still how cel-go supplies + // a late binding. + bindings := cel.Functions( + &functions.Overload{Operator: "instanceType_string", Unary: lookup(env.Instancetype, instancetypeZeroKeys)}, + &functions.Overload{Operator: "preset_string", Unary: lookup(env.Preset, nil)}, + ) + activation := map[string]any{valuesVar: defaulted} + + for _, c := range e.lets { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("let %q: %w", c.name, err) + } + activation[c.name] = out + } + + result := ResourceList{} + for _, c := range e.resources { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + q, err := toQuantity(out) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + if q.Sign() < 0 { + return nil, fmt.Errorf("resources[%q]: negative reservation %s", c.name, q.String()) + } + if q.IsZero() { + continue + } + result[c.name] = q + } + return result, nil +} + +func (e *Evaluator) run(c compiled, bindings cel.ProgramOption, activation map[string]any) (ref.Val, error) { + prg, err := e.env.Program(c.ast, bindings, cel.CostLimit(costLimit)) + if err != nil { + return nil, err + } + out, _, err := prg.Eval(activation) + if err != nil { + return nil, err + } + return out, nil +} + +// instancetypeZeroKeys are present, at zero, in every instanceType() result, +// so that a declaration can read any class of a type without guarding it. +var instancetypeZeroKeys = []string{ + KeyCPU, KeyDedicatedCPU, KeyMemory, KeyOvercommittedMemory, + KeyHugepagesPrefix + "2Mi", KeyHugepagesPrefix + "1Gi", +} + +func lookup(resolve func(string) (ResourceList, error), zeroKeys []string) functions.UnaryOp { + return func(arg ref.Val) ref.Val { + name, ok := arg.(types.String) + if !ok { + return types.MaybeNoSuchOverloadErr(arg) + } + rl, err := resolve(string(name)) + if err != nil { + return types.WrapErr(err) + } + out := make(map[string]any, len(rl)+len(zeroKeys)) + for _, k := range zeroKeys { + out[k] = apiservercel.Quantity{Quantity: resource.NewQuantity(0, resource.DecimalSI)} + } + for k, v := range rl { + q := v.DeepCopy() + out[k] = apiservercel.Quantity{Quantity: &q} + } + return types.DefaultTypeAdapter.NativeToValue(out) + } +} + +func mulQuantity(q, n ref.Val) ref.Val { + qq, ok := q.(apiservercel.Quantity) + if !ok { + return types.MaybeNoSuchOverloadErr(q) + } + nn, ok := n.(types.Int) + if !ok { + return types.MaybeNoSuchOverloadErr(n) + } + out := qq.Quantity.DeepCopy() + if !out.Mul(int64(nn)) { + return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn)) + } + return apiservercel.Quantity{Quantity: &out} +} + +func toQuantity(v ref.Val) (resource.Quantity, error) { + switch t := v.(type) { + case apiservercel.Quantity: + return t.Quantity.DeepCopy(), nil + case types.Int: + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Uint: + if uint64(t) > math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%d overflows a quantity", uint64(t)) + } + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Double: + f := float64(t) + if math.IsNaN(f) || math.IsInf(f, 0) { + return resource.Quantity{}, fmt.Errorf("not a finite number: %v", f) + } + // Rounding up would carry a negative figure smaller than a milli to + // zero, past the sign check. + if f < 0 { + return resource.Quantity{}, fmt.Errorf("negative reservation %v", f) + } + // float64(math.MaxInt64) rounds up to 2^63, so the bound is + // exclusive; past it the conversion would saturate into a smaller + // figure. + milli := math.Ceil(f * 1000) + if milli >= math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%v overflows a quantity", f) + } + return *resource.NewMilliQuantity(int64(milli), resource.DecimalSI), nil + case types.String: + q, err := resource.ParseQuantity(string(t)) + if err != nil { + return resource.Quantity{}, fmt.Errorf("parse %q as a quantity: %w", string(t), err) + } + return q, nil + default: + return resource.Quantity{}, fmt.Errorf("expression returned %s, expected an int, a double, a string or a quantity", v.Type().TypeName()) + } +} + +// checkTypes refuses a value whose JSON type the schema does not allow. It +// runs on normalized values: an integer is an int64, any other number a +// float64. A null is left to defaulting, which treats it as absent. +func checkTypes(v any, s *structuralschema.Structural, path string) error { + if s == nil || v == nil { + return nil + } + if s.XIntOrString { + switch v.(type) { + case int64, string: + return nil + } + return typeError(path, "an integer or a string", v) + } + // An untyped schema is defaulted as the shape its properties or items + // imply, so it is checked as that shape too. + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + } + } + switch effType { + case "object": + m, ok := v.(map[string]any) + if !ok { + return typeError(path, "an object", v) + } + for k, vv := range m { + ps, known := s.Properties[k] + switch { + case known: + if err := checkTypes(vv, &ps, path+"."+k); err != nil { + return err + } + case s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil: + if err := checkTypes(vv, s.AdditionalProperties.Structural, path+"."+k); err != nil { + return err + } + } + } + case "array": + sl, ok := v.([]any) + if !ok { + return typeError(path, "an array", v) + } + for i, vv := range sl { + if err := checkTypes(vv, s.Items, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + case "string": + if _, ok := v.(string); !ok { + return typeError(path, "a string", v) + } + case "integer": + if _, ok := v.(int64); !ok { + return typeError(path, "an integer", v) + } + case "number": + switch v.(type) { + case int64, float64: + default: + return typeError(path, "a number", v) + } + case "boolean": + if _, ok := v.(bool); !ok { + return typeError(path, "a boolean", v) + } + } + return nil +} + +func typeError(path, want string, got any) error { + if path == "" { + path = "." + } + return fmt.Errorf("values%s: expected %s, got %T", path, want, got) +} + +// normalizeNumbers returns a copy of v in which every JSON number is an int64 +// when it is integral and a float64 otherwise. Without it every number in a +// stored spec reaches CEL as a double, and replicas * quantity(...) would not +// type-check at runtime. +func normalizeNumbers(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, vv := range t { + out[k] = normalizeNumbers(vv) + } + return out + case []any: + out := make([]any, len(t)) + for i, vv := range t { + out[i] = normalizeNumbers(vv) + } + return out + case json.Number: + if i, err := t.Int64(); err == nil { + return i + } + // 2.0 and 2e0 are integers to the schema validator; spelled that + // way they fail Int64 and are classified as a float64 would be. + f, _ := t.Float64() + return normalizeNumbers(f) + case float64: + if t == math.Trunc(t) && math.Abs(t) < 1<<53 { + return int64(t) + } + return t + case int: + return int64(t) + default: + return v + } +} diff --git a/pkg/reservation/evaluator_test.go b/pkg/reservation/evaluator_test.go new file mode 100644 index 000000000..7aa7ef9a7 --- /dev/null +++ b/pkg/reservation/evaluator_test.go @@ -0,0 +1,525 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "os" + "sync" + "testing" + + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// The declarations below are test fixtures in the shape a kind would carry +// on its ApplicationDefinition. They are evaluated against the kinds' real +// shipped schemas so that defaulting is exercised with the defaults tenants +// actually get. + +var postgresDecl = &Declaration{ + Let: []Binding{{ + Name: "count", + // The autoscaler writes the Cluster's scale subresource, never the + // application, so the ceiling is what has to be reserved. + Expression: `values.autoscaling.enabled && values.autoscaling.maxReplicas > values.replicas + ? values.autoscaling.maxReplicas : values.replicas`, + }}, + Resources: map[string]string{ + // cozy-lib fills a preset per key, so each key is overridden on its own. + "cpu": `(has(values.resources.cpu) ? quantity(values.resources.cpu) : preset(values.resourcesPreset).cpu).mul(count)`, + "memory": `(has(values.resources.memory) ? quantity(values.resources.memory) : preset(values.resourcesPreset).memory).mul(count)`, + "storage": `quantity(values.size).mul(count)`, + }, +} + +var vmInstanceDecl = &Declaration{ + Let: []Binding{ + // vm-instance treats resources as all-or-nothing, and cpu is per socket. + {Name: "sized", Expression: `has(values.resources.cpu) && has(values.resources.sockets) && has(values.resources.memory)`}, + {Name: "it", Expression: `instanceType(values.instanceType)`}, + }, + Resources: map[string]string{ + "cpu": `sized ? int(values.resources.cpu) * int(values.resources.sockets) : it.cpu`, + "dedicated-cpu": `sized ? 0 : it['dedicated-cpu']`, + "memory": `sized ? quantity(values.resources.memory) : it.memory`, + "hugepages-2Mi": `sized ? 0 : it['hugepages-2Mi']`, + "hugepages-1Gi": `sized ? 0 : it['hugepages-1Gi']`, + "overcommitted-memory": `sized ? 0 : it['overcommitted-memory']`, + "services.loadbalancers": `values.external ? 1 : 0`, + }, +} + +var kubernetesNodesDecl = &Declaration{ + Let: []Binding{{Name: "it", Expression: `instanceType(values.instanceType)`}}, + Resources: map[string]string{ + "cpu": `it.cpu.mul(values.maxReplicas)`, + "memory": `it.memory.mul(values.maxReplicas)`, + }, +} + +func shippedSchema(t *testing.T, kind string) *structuralschema.Structural { + t.Helper() + raw, err := os.ReadFile("../../packages/system/" + kind + "-rd/cozyrds/" + kind + ".yaml") + if err != nil { + t.Fatal(err) + } + var def struct { + Spec struct { + Application struct { + OpenAPISchema string `json:"openAPISchema"` + } `json:"application"` + } `json:"spec"` + } + if err := yaml.Unmarshal(raw, &def); err != nil { + t.Fatal(err) + } + s, err := defaulting.BuildSchema(def.Spec.Application.OpenAPISchema) + if err != nil || s == nil { + t.Fatalf("%s schema: %v", kind, err) + } + return s +} + +func testEnvironment(t *testing.T) Environment { + t.Helper() + env, err := NewStaticEnvironment(shippedInstancetypes(t), map[string]ResourceList{ + "t1.nano": {"cpu": resource.MustParse("250m"), "memory": resource.MustParse("128Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + "t1.micro": {"cpu": resource.MustParse("500m"), "memory": resource.MustParse("256Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + }) + if err != nil { + t.Fatal(err) + } + return env +} + +func mustCompile(t *testing.T, decl *Declaration, kind string) *Evaluator { + t.Helper() + ev, err := Compile(decl, shippedSchema(t, kind)) + if err != nil { + t.Fatalf("compile %s: %v", kind, err) + } + return ev +} + +func rl(kv ...string) ResourceList { + out := ResourceList{} + for i := 0; i < len(kv); i += 2 { + out[kv[i]] = resource.MustParse(kv[i+1]) + } + return out +} + +func TestEvaluateKinds(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + cases := []struct { + name string + ev *Evaluator + values string + want ResourceList + }{ + { + // Two replicas of t1.micro with 10Gi each, all from schema defaults. + name: "postgres defaults", ev: postgres, values: `{}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + // The per-key fill: cpu from resources, memory from the preset. + name: "postgres partial resources on a preset", ev: postgres, + values: `{"replicas": 1, "resourcesPreset": "t1.nano", "resources": {"cpu": "1"}}`, + want: rl("cpu", "1", "memory", "128Mi", "storage", "10Gi"), + }, + { + name: "postgres integer cpu", ev: postgres, + values: `{"replicas": 1, "resources": {"cpu": 2, "memory": "1Gi"}}`, + want: rl("cpu", "2", "memory", "1Gi", "storage", "10Gi"), + }, + { + name: "postgres autoscaling reserves the ceiling", ev: postgres, + values: `{"autoscaling": {"enabled": true}}`, + want: rl("cpu", "3", "memory", "1536Mi", "storage", "60Gi"), + }, + { + name: "postgres autoscaling off reserves replicas", ev: postgres, + values: `{"autoscaling": {"enabled": false, "maxReplicas": 6}}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + name: "vm-instance default instance type", ev: vm, values: `{}`, + want: rl("cpu", "1", "memory", "4Gi"), + }, + { + name: "vm-instance dedicated type pins the emulator core", ev: vm, + values: `{"instanceType": "cx1.2xlarge"}`, + want: rl("dedicated-cpu", "9", "hugepages-2Mi", "16Gi"), + }, + { + name: "vm-instance overcommitted type", ev: vm, + values: `{"instanceType": "o1.large"}`, + want: rl("cpu", "2", "overcommitted-memory", "8Gi"), + }, + { + // cpu is cores per socket. + name: "vm-instance complete resources block", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "sockets": 2, "memory": "8Gi"}}`, + want: rl("cpu", "8", "memory", "8Gi"), + }, + { + // Without sockets the chart sizes the VM from its instance type. + name: "vm-instance cpu without sockets", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "memory": "8Gi"}}`, + want: rl("cpu", "1", "memory", "2Gi"), + }, + { + name: "vm-instance external", ev: vm, + values: `{"instanceType": "u1.small", "external": true}`, + want: rl("cpu", "1", "memory", "2Gi", "services.loadbalancers", "1"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.ev.EvaluateJSON([]byte(tc.values), env) + if err != nil { + t.Fatal(err) + } + assertList(t, tc.name, got, tc.want) + }) + } +} + +// A release a platform chart renders, such as ComputePlane's default md0 +// pool, carries only the keys the chart wrote. It has to reserve exactly what +// the same pool with its defaults written out reserves. +func TestEvaluateDefaultsStoredValues(t *testing.T) { + env := testEnvironment(t) + ev := mustCompile(t, kubernetesNodesDecl, "kubernetes-nodes") + + sparse, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0}`), env) + if err != nil { + t.Fatal(err) + } + explicit, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0, "maxReplicas": 10, "instanceType": "u1.medium"}`), env) + if err != nil { + t.Fatal(err) + } + assertList(t, "sparse vs explicit", sparse, explicit) + assertList(t, "sparse", sparse, rl("cpu", "10", "memory", "40Gi")) +} + +func TestEvaluateFailsClosed(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + if _, err := vm.EvaluateJSON([]byte(`{"instanceType": "u9.imaginary"}`), env); err == nil { + t.Error("unknown instance type: expected an error") + } else if !errors.Is(err, ErrNotFound) { + t.Errorf("unknown instance type: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"resourcesPreset": "t9.huge"}`), env); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown preset: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"replicas": "two"}`), env); err == nil { + t.Error("wrong value type: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{"size": "lots"}`), env); err == nil { + t.Error("unparseable quantity: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`[1, 2]`), env); err == nil { + t.Error("non-object values: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{}`), nil); err == nil { + t.Error("no environment: expected an error") + } + + unguarded, err := Compile(&Declaration{Resources: map[string]string{"cpu": `quantity(values.notInTheSchema)`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + if _, err := unguarded.EvaluateJSON([]byte(`{}`), env); err == nil { + t.Error("unguarded absent key: expected an error, not a zero") + } + + negative, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-1`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := negative.EvaluateJSON(nil, env); err == nil { + t.Error("negative reservation: expected an error") + } + + // A negative double too small to survive rounding to millis is still + // negative. + tiny, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-0.0001`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tiny.EvaluateJSON(nil, env); err == nil { + t.Error("small negative reservation: expected an error, not a zero") + } + + // A double past the range of an int64 in millis must not be clamped into + // a smaller figure. + for _, expr := range []string{`1e100`, `1e308`, `9.3e15`} { + huge, err := Compile(&Declaration{Resources: map[string]string{"cpu": expr}}, nil) + if err != nil { + t.Fatal(err) + } + if got, err := huge.EvaluateJSON(nil, env); err == nil { + t.Errorf("%s: expected an error, got %v", expr, got) + } + } + finite, err := Compile(&Declaration{Resources: map[string]string{"cpu": `1e3`, "memory": `0.25`}}, nil) + if err != nil { + t.Fatal(err) + } + got, err := finite.EvaluateJSON(nil, env) + if err != nil { + t.Fatal(err) + } + assertList(t, "finite doubles", got, rl("cpu", "1k", "memory", "250m")) +} + +func TestEvaluateRejectsValuesOfTheWrongType(t *testing.T) { + env := testEnvironment(t) + // Defaulting replaces a value whose type does not match the schema with + // the schema's own default, so the type is checked before it runs: + // otherwise this guard would read an erased resources object and + // reserve its fallback. + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{ + `{"resources": "invalid"}`, + `{"resources": [1]}`, + `{"resources": true}`, + `{"resources": {"cpu": true}}`, + `{"replicas": 2.5}`, + `{"replicas": "2"}`, + `{"external": "yes"}`, + `{"postgresql": {"sharedPreloadLibraries": "pg_prewarm"}}`, + `{"users": {"app": "admin"}}`, + } { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{}`: rl("cpu", "1"), + `{"resources": {}}`: rl("cpu", "1"), + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"resources": {"cpu": "500m"}}`: rl("cpu", "500m"), + `{"resources": null, "replicas": 3}`: rl("cpu", "1"), + `{"users": {"app": {"replication": true}}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestEvaluateAcceptsIntegralSpellings(t *testing.T) { + env := testEnvironment(t) + // The Kubernetes schema validator accepts an integral number however it + // is spelled, so the evaluator must too. + ev, err := Compile(&Declaration{Resources: map[string]string{"cpu": `values.replicas`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"replicas": 2}`, `{"replicas": 2.0}`, `{"replicas": 2e0}`, `{"replicas": 0.2e1}`} { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, rl("cpu", "2")) + } + for _, raw := range []string{`{"replicas": 2.5}`, `{"replicas": 25e-1}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } +} + +func TestEvaluateChecksUntypedSchemasByShape(t *testing.T) { + env := testEnvironment(t) + // BuildSchema does not require a structural schema to be typed, and + // defaulting treats an untyped node with properties as an object and one + // with items as an array, so the type check must as well. + s, err := defaulting.BuildSchema(`{"type":"object","properties":{ + "resources":{"properties":{"cpu":{"type":"integer"}}}, + "disks":{"items":{"type":"string"}}, + "labels":{"additionalProperties":{"type":"string"}}}}`) + if err != nil { + t.Fatal(err) + } + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources) && has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, s) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"resources": "invalid"}`, `{"resources": {"cpu": "2"}}`, `{"disks": "a"}`, `{"disks": [1]}`, `{"labels": "a"}`, `{"labels": {"a": 1}}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"disks": ["a"]}`: rl("cpu", "1"), + `{"labels": {"a": "b"}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestCompile(t *testing.T) { + if _, err := Compile(nil, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("nil declaration: got %v, want ErrNoDeclaration", err) + } + if _, err := Compile(&Declaration{}, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("empty declaration: got %v, want ErrNoDeclaration", err) + } + + exempt, err := Compile(&Declaration{Exempt: true}, nil) + if err != nil { + t.Fatal(err) + } + got, err := exempt.EvaluateJSON([]byte(`{"anything": 1}`), nil) + if err != nil || len(got) != 0 || !exempt.Exempt() { + t.Errorf("exempt: got %v, %v", got, err) + } + + invalid := map[string]*Declaration{ + "exempt with resources": {Exempt: true, Resources: map[string]string{"cpu": "1"}}, + "syntax error": {Resources: map[string]string{"cpu": "values.("}}, + "undeclared variable": {Resources: map[string]string{"cpu": "count"}}, + "empty key": {Resources: map[string]string{"": "1"}}, + "bad let name": {Let: []Binding{{Name: "a-b", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "let shadows values": {Let: []Binding{{Name: "values", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "duplicate let": {Let: []Binding{{Name: "a", Expression: "1"}, {Name: "a", Expression: "2"}}, Resources: map[string]string{"cpu": "a"}}, + } + for name, decl := range invalid { + if _, err := Compile(decl, nil); err == nil { + t.Errorf("%s: expected a compile error", name) + } + } +} + +func TestEvaluateIsConcurrencySafe(t *testing.T) { + env := testEnvironment(t) + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + catalog := shippedInstancetypes(t) + var wg sync.WaitGroup + for _, it := range []string{"u1.small", "cx1.2xlarge", "o1.large", "u1.medium"} { + want, err := ClassifyInstancetype(findInstancetype(t, catalog, it)) + if err != nil { + t.Fatal(err) + } + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + got, err := vm.EvaluateJSON([]byte(`{"instanceType": "`+it+`"}`), env) + if err != nil { + t.Error(err) + return + } + assertList(t, it, got, want) + }() + } + } + wg.Wait() +} + +func findInstancetype(t *testing.T, catalog []map[string]any, name string) map[string]any { + t.Helper() + for _, obj := range catalog { + if obj["metadata"].(map[string]any)["name"] == name { + return obj + } + } + t.Fatalf("%s is not shipped", name) + return nil +} + +func TestAggregate(t *testing.T) { + env := testEnvironment(t) + evaluators := map[string]*Evaluator{ + "VMInstance": mustCompile(t, vmInstanceDecl, "vm-instance"), + "Postgres": mustCompile(t, postgresDecl, "postgres"), + } + lookup := func(kind, group string) (*Evaluator, error) { + if group != "apps.cozystack.io" { + return nil, errors.New("unknown group") + } + if ev, ok := evaluators[kind]; ok { + return ev, nil + } + return nil, ErrNoDeclaration + } + apps := []Application{ + {Namespace: "tenant-a", Name: "vm1", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-a", Name: "vm2", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small", "external": true}`)}, + {Namespace: "tenant-a", Name: "db", Kind: "Postgres", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + {Namespace: "tenant-b", Name: "vm3", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-b", Name: "broken", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "gone"}`)}, + {Namespace: "tenant-b", Name: "undeclared", Kind: "Redis", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + } + sums, failures := Aggregate(apps, lookup, env) + assertList(t, "tenant-a", sums["tenant-a"], rl("cpu", "3", "memory", "4608Mi", "storage", "20Gi", "services.loadbalancers", "1")) + assertList(t, "tenant-b", sums["tenant-b"], rl("cpu", "1", "memory", "2Gi")) + if len(failures) != 2 { + t.Fatalf("expected 2 failures, got %v", failures) + } + if failures[0].Application.Name != "broken" || !errors.Is(failures[0].Err, ErrNotFound) { + t.Errorf("first failure: %v", failures[0]) + } + if failures[1].Application.Name != "undeclared" || !errors.Is(failures[1].Err, ErrNoDeclaration) { + t.Errorf("second failure: %v", failures[1]) + } +} + +func TestResourceListArithmetic(t *testing.T) { + a := rl("cpu", "2", "memory", "4Gi") + b := a.DeepCopy() + b.Add(rl("cpu", "1", "storage", "10Gi")) + assertList(t, "copy unaffected", a, rl("cpu", "2", "memory", "4Gi")) + assertList(t, "add", b, rl("cpu", "3", "memory", "4Gi", "storage", "10Gi")) + b.Sub(rl("cpu", "4", "services.loadbalancers", "1")) + assertList(t, "sub", b, rl("cpu", "-1", "memory", "4Gi", "storage", "10Gi", "services.loadbalancers", "-1")) + if keys := b.Keys(); len(keys) != 4 || keys[0] != "cpu" || keys[3] != "storage" { + t.Errorf("keys not sorted: %v", keys) + } +} diff --git a/pkg/reservation/instancetype.go b/pkg/reservation/instancetype.go new file mode 100644 index 000000000..b4373164f --- /dev/null +++ b/pkg/reservation/instancetype.go @@ -0,0 +1,148 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// EmulatorThreadCompleteToEvenParityAnnotation, set in an instance type's +// spec.annotations, makes KubeVirt give the isolated emulator thread one or +// two cores, whichever makes the pinned total even. KubeVirt copies those +// annotations onto the VMI and tests the key for presence, so its value is +// irrelevant. +const EmulatorThreadCompleteToEvenParityAnnotation = "alpha.kubevirt.io/EmulatorThreadCompleteToEvenParity" + +// ClassifyInstancetype resolves a VirtualMachineClusterInstancetype into the +// class-keyed reservation of one VM of that type. +// +// The class is read from the type's spec, never from its name, so a type an +// operator adds classifies itself. Cores include the emulator and +// supplemental I/O threads a dedicated type pins, because those are fixed +// properties of the type a tenant can see before ordering; the launcher's +// memory overhead is computed by virt-controller at runtime and is platform +// margin, not part of the reservation. +func ClassifyInstancetype(obj map[string]any) (ResourceList, error) { + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + + guestCPU, err := nestedInt(obj, "spec", "cpu", "guest") + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest: %w", name, err) + } + if guestCPU <= 0 { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest must be positive, got %d", name, guestCPU) + } + guestMemoryRaw, found, err := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "guest") + if err != nil || !found { + return nil, fmt.Errorf("instance type %q: spec.memory.guest is required", name) + } + guestMemory, err := parseQuantity(guestMemoryRaw) + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.memory.guest: %w", name, err) + } + + out := ResourceList{} + + dedicated, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "dedicatedCPUPlacement") + if dedicated { + cores := guestCPU + if isolate, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "isolateEmulatorThread"); isolate { + cores++ + annotations, _, _ := unstructured.NestedMap(obj, "spec", "annotations") + if _, parity := annotations[EmulatorThreadCompleteToEvenParityAnnotation]; parity && cores%2 == 1 { + cores++ + } + } + if policy, _, _ := unstructured.NestedString(obj, "spec", "ioThreadsPolicy"); policy == "supplementalPool" { + extra, err := nestedInt(obj, "spec", "ioThreads", "supplementalPoolThreadCount") + if err != nil { + return nil, fmt.Errorf("instance type %q: ioThreadsPolicy supplementalPool: spec.ioThreads.supplementalPoolThreadCount: %w", name, err) + } + cores += extra + } + out[KeyDedicatedCPU] = *resource.NewQuantity(cores, resource.DecimalSI) + } else { + out[KeyCPU] = *resource.NewQuantity(guestCPU, resource.DecimalSI) + } + + pageSize, hasHugepages, _ := unstructured.NestedString(obj, "spec", "memory", "hugepages", "pageSize") + overcommit, hasOvercommit, _ := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "overcommitPercent") + overcommitted := hasOvercommit && overcommit != nil && fmt.Sprint(overcommit) != "0" + switch { + case hasHugepages && overcommitted: + return nil, fmt.Errorf("instance type %q: hugepages and overcommitPercent are mutually exclusive", name) + case hasHugepages: + if pageSize == "" { + return nil, fmt.Errorf("instance type %q: spec.memory.hugepages.pageSize is empty", name) + } + out[KeyHugepagesPrefix+pageSize] = guestMemory + case overcommitted: + // The launcher requests less than the guest here, but the tenant + // declared the guest figure, and the same page cannot be granted as + // both overcommitted and ordinary memory. + out[KeyOvercommittedMemory] = guestMemory + default: + out[KeyMemory] = guestMemory + } + return out, nil +} + +func nestedInt(obj map[string]any, fields ...string) (int64, error) { + v, found, err := unstructured.NestedFieldNoCopy(obj, fields...) + if err != nil { + return 0, err + } + if !found || v == nil { + return 0, fmt.Errorf("is required") + } + switch n := v.(type) { + case int64: + return n, nil + case int: + return int64(n), nil + case int32: + return int64(n), nil + case float64: + if n != float64(int64(n)) { + return 0, fmt.Errorf("must be an integer, got %v", n) + } + return int64(n), nil + default: + return 0, fmt.Errorf("must be an integer, got %T", v) + } +} + +func parseQuantity(v any) (resource.Quantity, error) { + switch q := v.(type) { + case string: + return resource.ParseQuantity(q) + case int64: + return *resource.NewQuantity(q, resource.BinarySI), nil + case int: + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + case float64: + if q != float64(int64(q)) { + return resource.Quantity{}, fmt.Errorf("fractional byte count %v", q) + } + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + default: + return resource.Quantity{}, fmt.Errorf("must be a quantity, got %T", v) + } +} diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go new file mode 100644 index 000000000..0d072b548 --- /dev/null +++ b/pkg/reservation/instancetype_test.go @@ -0,0 +1,259 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" +) + +// shippedInstancetypes decodes every VirtualMachineClusterInstancetype the +// platform ships, so that the classification is pinned against the real +// catalog rather than against hand-written fixtures. +func shippedInstancetypes(t *testing.T) []map[string]any { + t.Helper() + files, err := filepath.Glob("../../packages/system/kubevirt-instancetypes/templates/*.yaml") + if err != nil || len(files) == 0 { + t.Fatalf("no instance type templates found: %v", err) + } + var out []map[string]any + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + t.Fatal(err) + } + dec := utilyaml.NewYAMLOrJSONDecoder(fh, 4096) + for { + var obj map[string]any + if err := dec.Decode(&obj); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("%s: %v", f, err) + } + if obj["kind"] == "VirtualMachineClusterInstancetype" { + out = append(out, obj) + } + } + fh.Close() + } + return out +} + +func TestClassifyShippedInstancetypes(t *testing.T) { + // The expected class per series is written from what each series is, + // not derived from the spec, so a name-based rule and a spec-based rule + // disagreeing shows up here. m1 is shared CPU with hugepages, n1 a + // dedicated series with hugepages, o1 overcommits memory. + type class struct{ cpu, memory string } + want := map[string]class{ + "u1": {KeyCPU, KeyMemory}, + "gn1": {KeyCPU, KeyMemory}, + "m1": {KeyCPU, "hugepages"}, + "o1": {KeyCPU, KeyOvercommittedMemory}, + "d1": {KeyDedicatedCPU, KeyMemory}, + "cx1": {KeyDedicatedCPU, "hugepages"}, + "n1": {KeyDedicatedCPU, "hugepages"}, + "rt1": {KeyDedicatedCPU, "hugepages"}, + } + + types := shippedInstancetypes(t) + if len(types) < 60 { + t.Fatalf("expected the full shipped catalog, decoded only %d instance types", len(types)) + } + seen := map[string]bool{} + for _, obj := range types { + name := obj["metadata"].(map[string]any)["name"].(string) + series := strings.SplitN(name, ".", 2)[0] + seen[series] = true + w, ok := want[series] + if !ok { + t.Errorf("%s: series %q has no expected class; add it to this table", name, series) + continue + } + rl, err := ClassifyInstancetype(obj) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + if len(rl) != 2 { + t.Errorf("%s: expected one CPU class and one memory class, got %v", name, rl) + } + if _, ok := rl[w.cpu]; !ok { + t.Errorf("%s: expected CPU class %q, got %v", name, w.cpu, rl) + } + if w.memory == "hugepages" { + found := false + for k := range rl { + found = found || strings.HasPrefix(k, KeyHugepagesPrefix) + } + if !found { + t.Errorf("%s: expected a hugepages class, got %v", name, rl) + } + } else if _, ok := rl[w.memory]; !ok { + t.Errorf("%s: expected memory class %q, got %v", name, w.memory, rl) + } + } + for series := range want { + if !seen[series] { + t.Errorf("series %q is no longer shipped; drop it from this table", series) + } + } +} + +func TestClassifyShippedFigures(t *testing.T) { + byName := map[string]map[string]any{} + for _, obj := range shippedInstancetypes(t) { + byName[obj["metadata"].(map[string]any)["name"].(string)] = obj + } + cases := []struct { + name string + want ResourceList + }{ + // The KubeVirt u1.small is 2Gi; the cozy-lib preset of the same name + // is 4Gi. The two catalogs share names, not figures. + {"u1.small", ResourceList{KeyCPU: resource.MustParse("1"), KeyMemory: resource.MustParse("2Gi")}}, + // Eight guest vCPUs on nine pinned cores: the isolated emulator + // thread takes a core nobody else can be given. + {"cx1.2xlarge", ResourceList{KeyDedicatedCPU: resource.MustParse("9"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + {"d1.large", ResourceList{KeyDedicatedCPU: resource.MustParse("3"), KeyMemory: resource.MustParse("8Gi")}}, + {"m1.large", ResourceList{KeyCPU: resource.MustParse("2"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + } + for _, tc := range cases { + obj, ok := byName[tc.name] + if !ok { + t.Fatalf("%s is not shipped", tc.name) + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + assertList(t, tc.name, got, tc.want) + } +} + +func TestClassifyEmulatorThreadParity(t *testing.T) { + // KubeVirt copies an instance type's spec.annotations onto the VMI and + // pins the extra parity core when the VMI carries the annotation, so the + // figure is a property of each type, not of the catalog. + dedicated := func(guest int64, annotations map[string]any) map[string]any { + spec := map[string]any{ + "cpu": map[string]any{"guest": guest, "dedicatedCPUPlacement": true, "isolateEmulatorThread": true}, + "memory": map[string]any{"guest": "1Gi"}, + } + if annotations != nil { + spec["annotations"] = annotations + } + return map[string]any{"metadata": map[string]any{"name": "test"}, "spec": spec} + } + parity := map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: ""} + cases := []struct { + guest int64 + annotations map[string]any + want int64 + }{ + {guest: 8, want: 9}, + {guest: 8, annotations: parity, want: 10}, + {guest: 8, annotations: map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: "false"}, want: 10}, + {guest: 8, annotations: map[string]any{"example.com/unrelated": ""}, want: 9}, + {guest: 3, want: 4}, + {guest: 3, annotations: parity, want: 4}, + } + for _, tc := range cases { + got, err := ClassifyInstancetype(dedicated(tc.guest, tc.annotations)) + if err != nil { + t.Fatal(err) + } + cores := got[KeyDedicatedCPU] + if cores.Value() != tc.want { + t.Errorf("guest %d annotations %v: got %s dedicated cores, want %d", tc.guest, tc.annotations, cores.String(), tc.want) + } + } + + // Only the type's own annotations reach the VMI; metadata.annotations + // describe the object and are not copied. + onMetadata := dedicated(8, nil) + onMetadata["metadata"].(map[string]any)["annotations"] = parity + got, err := ClassifyInstancetype(onMetadata) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 9 { + t.Errorf("annotation on metadata: got %s dedicated cores, want 9", cores.String()) + } +} + +func TestClassifySupplementalIOThreads(t *testing.T) { + obj := map[string]any{ + "metadata": map[string]any{"name": "test"}, + "spec": map[string]any{ + "cpu": map[string]any{"guest": int64(4), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "ioThreads": map[string]any{"supplementalPoolThreadCount": int64(2)}, + "memory": map[string]any{"guest": "1Gi"}, + }, + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 6 { + t.Errorf("got %s dedicated cores, want 6", cores.String()) + } +} + +func TestClassifyRejectsIncompleteTypes(t *testing.T) { + cases := map[string]map[string]any{ + "no guest cpu": {"spec": map[string]any{"memory": map[string]any{"guest": "1Gi"}}}, + "no guest memory": {"spec": map[string]any{"cpu": map[string]any{"guest": int64(1)}}}, + "hugepages and overcommit": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1)}, + "memory": map[string]any{"guest": "1Gi", "overcommitPercent": int64(50), "hugepages": map[string]any{"pageSize": "2Mi"}}, + }}, + "supplementalPool without a count": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "memory": map[string]any{"guest": "1Gi"}, + }}, + } + for name, obj := range cases { + if got, err := ClassifyInstancetype(obj); err == nil { + t.Errorf("%s: expected an error, got %v", name, got) + } + } +} + +func assertList(t *testing.T, label string, got, want ResourceList) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s: got %v, want %v", label, got, want) + return + } + for k, w := range want { + g, ok := got[k] + if !ok || g.Cmp(w) != 0 { + t.Errorf("%s: %s = %s, want %s (full: %v)", label, k, g.String(), w.String(), got) + } + } +} diff --git a/pkg/reservation/types.go b/pkg/reservation/types.go new file mode 100644 index 000000000..1a01a0763 --- /dev/null +++ b/pkg/reservation/types.go @@ -0,0 +1,113 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package reservation computes what an application consumes from a tenant's +// quota budget, from its declared values, before anything runs. +// +// A reservation is denominated in the shorthand keys a Tenant's +// resourceQuotas uses (cpu, memory, storage, services.loadbalancers, ...), +// never in rendered pod units: no allocation ratio is applied and no +// virtualization overhead is added. The evaluator holds no per-kind +// knowledge; each kind describes its consumption in a Declaration carried by +// its ApplicationDefinition, so a kind shipped from an out-of-tree catalog +// participates without a change here. +package reservation + +import ( + "sort" + + "k8s.io/apimachinery/pkg/api/resource" +) + +// Quota keys for KubeVirt instance-type classes. A shared vCPU and a pinned +// core are not the same capacity, nor are ordinary, hugepage-backed and +// overcommitted memory, so each class counts against its own key. +const ( + KeyCPU = "cpu" + KeyDedicatedCPU = "dedicated-cpu" + KeyMemory = "memory" + KeyOvercommittedMemory = "overcommitted-memory" + // KeyHugepagesPrefix is followed by the page size, e.g. "hugepages-2Mi", + // matching the corev1 resource name for the same pages. + KeyHugepagesPrefix = "hugepages-" +) + +// Declaration is how a kind's reservation is computed from its values. +// +// Exactly one of Exempt or Resources is meaningful: a kind that consumes +// nothing says so with Exempt rather than with an empty Resources map, so +// that a zero reservation is always a reviewed statement and never an +// omission. +type Declaration struct { + // Exempt declares that the kind consumes nothing from a tenant's budget. + Exempt bool `json:"exempt,omitempty"` + + // Let binds intermediate CEL expressions, evaluated in order, each one + // visible to the later bindings and to every Resources expression. + Let []Binding `json:"let,omitempty"` + + // Resources maps a quota key to the CEL expression computing the + // amount of it the application reserves. An expression may return an + // int, a double, a quantity string or a quantity. + Resources map[string]string `json:"resources,omitempty"` +} + +// Binding names one intermediate CEL expression of a Declaration. +type Binding struct { + Name string `json:"name"` + Expression string `json:"expression"` +} + +// ResourceList is a reservation, keyed by quota key. +type ResourceList map[string]resource.Quantity + +// Add adds other into l in place. +func (l ResourceList) Add(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Add(v) + l[k] = cur + } +} + +// Sub subtracts other from l in place. A key absent from l goes negative, +// which is what the delta of an Update needs. +func (l ResourceList) Sub(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Sub(v) + l[k] = cur + } +} + +// DeepCopy returns an independent copy of l. +func (l ResourceList) DeepCopy() ResourceList { + out := make(ResourceList, len(l)) + for k, v := range l { + out[k] = v.DeepCopy() + } + return out +} + +// Keys returns the keys of l in a stable order, for deterministic messages. +func (l ResourceList) Keys() []string { + keys := make([]string, 0, len(l)) + for k := range l { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}