cozystack/hack/issue-triage-contract.bats

#!/usr/bin/env bats

# Contract for the issue triage labeler. Like promote-gate-contract.bats and
# release-freeze-contract.bats, these tests pin executable/structural workflow
# lines rather than prose: a guard demoted to a comment must never satisfy the
# contract. The workflow's own comments name every API and constant asserted
# below, so the JS block is filtered on `//` as well as `#`.
#
# No PR lane can exercise this workflow. It runs on `issues`, on a daily cron
# and on workflow_dispatch, so the first real execution is a sweep over every
# open issue in the repository, writing a label to each. The lines pinned here
# are the ones that bound that blast radius: the skip that keeps pull requests
# out of it, the skip that keeps an already-triaged issue out of it, the pacing
# and the per-run cap that keep the sweep inside GitHub's secondary rate limits,
# the pinned action, and the token scopes. Each is a single deletion away from a
# repository-wide mess that nothing reports — drop the pull-request skip and
# every open PR is told it needs triage; drop the pacing and the sweep is
# throttled and dies half done.
#
# The rate-limit tests assert the invariant (stay under GitHub's published
# ceilings) rather than today's constants, so tuning the pacing within the safe
# band does not need a test edit, while leaving it does go red.
#
# Known ceiling of the approach: these are structural greps, so they prove a
# guard is present and executable, not that it is reachable. Wrapping the block
# around a guard in `if (false)` satisfies every pin here, because each matches
# the guard's own line and nothing above it; editing the pinned line itself does
# not, so `if (false && issue.pull_request)` goes red. Behavioural coverage of the script
# would need a JS runtime, and `checks` runs where none is guaranteed, so a
# harness would go silently green wherever node is missing.

load test_helper

REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
WF="$REPO_ROOT/.github/workflows/issue-triage.yaml"
STALE="$REPO_ROOT/.github/workflows/stale.yaml"
LABELS="$REPO_ROOT/.github/labels.yml"

# GitHub's documented secondary rate limits for content-generating requests.
# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api
WRITES_PER_MINUTE=80
WRITES_PER_HOUR=500

# Strip YAML comments. POSIX `grep` only: the unit-test runner has no ripgrep.
# grep exits 1 when nothing is selected (legitimate for an all-comment block)
# and 2 on a real error. Do not read the `rc` check below as what catches that,
# here or in script_lines: neither helper is ever the last stage of a pipeline
# in this file, so ordinary (non-pipefail) pipe semantics discard whatever it
# returns, and `hack/cozytest.sh`'s translator appends `return 0` to every line
# that is exactly `}`, both helpers' closing braces included. What catches a
# real grep error is the OUTPUT, and only for one class of pin: the error
# empties the stream, so a pin requiring something to be present in it fails on
# that. A pin demanding an absence stays blind, because an emptied stream
# produces the same count of zero the pin was written to assert. The
# `[ -n "$block" ]` guard at the top of a test is not that check either: it
# runs on the raw awk extraction, upstream of both helpers, and cannot see
# either of them fail.
code_lines() {
  local rc=0
  grep -v '^[[:space:]]*#' || rc=$?
  [ "$rc" -le 1 ]
}

# Strip JavaScript comments. The labeling logic lives in a github-script
# `script: |` block whose comments discuss the pull_request skip, the cap and
# the pacing by name, so filtering on `#` alone would let a comment satisfy a
# pin.
script_lines() {
  local rc=0
  grep -v '^[[:space:]]*//' || rc=$?
  [ "$rc" -le 1 ]
}

# Body of one job, up to the next top-level job key.
job_block() {
  awk -v job="  $1:" '
    $0 == job { inside = 1; next }
    /^  [a-z0-9_-]+:$/ { inside = 0 }
    inside' "$2"
}

# Integer on the right of the first `=` on the first matching line. Callers
# assert the plain-literal form first, because this strips non-digits rather
# than evaluating: `= 60 * 1000;` would read as 601000 and quietly satisfy a
# bound it never computed.
number_after_eq() {
  awk -F'=' 'NR == 1 { gsub(/[^0-9]/, "", $2); print $2 }'
}

# The declaration this file can read: one integer literal, no arithmetic.
literal_const() { # <name> <block>
  printf '%s\n' "$2" | script_lines | grep -qE "const $1 = [0-9]+;[[:space:]]*$"
}

# The accepted-signal list, one label per line.
signal_labels() {
  script_lines < "$WF" |
    awk '/const ACCEPTED_SIGNAL_LABELS = \[/ { inside = 1; next } /\];/ { inside = 0 } inside' |
    sed -n "s/^[[:space:]]*'\([^']*\)'.*/\1/p"
}

# Minutes past midnight for a 5-field cron, from its first two fields.
cron_minutes() {
  sed "s/^[^']*'//; s/'.*$//" | awk 'NR == 1 { print $2 * 60 + $1 }'
}

@test "the workflow under contract exists" {
  [ -f "$WF" ]
}

# ── what the sweep must not touch ────────────────────────────────────────────
# listForRepo returns pull requests alongside issues, and the sweep re-reads
# every open issue every day. Both skips below are the difference between a
# no-op pass and a repository-wide mislabeling that no run reports as an error.

@test "pull requests are skipped by the classifier" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # Losing this tells every open pull request it needs triage, on the first
  # sweep, with a green job.
  printf '%s\n' "$block" | script_lines | grep -qF 'if (issue.pull_request) return null'
}

@test "an issue that already carries a triage label is left alone" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # The sweep is additive and never removes a label, so without this skip a
  # daily pass re-adds a second, contradictory triage/* label to issues a
  # maintainer has already classified.
  printf '%s\n' "$block" | script_lines | grep -qF "labels.some((n) => n.startsWith('triage/'))"
}

# ── staying inside GitHub's secondary rate limits ────────────────────────────
# The sweep is the only bulk writer in the tree. The action's client carries no
# throttling plugin, its retry default is off, and a throttled write answers
# 403, which is exempt from retry anyway: an unpaced sweep does not slow down
# when it is throttled, it fails the job partway and leaves the backlog half
# labeled.

@test "writes are paced under the per-minute ceiling" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # The wait has to be executed, not merely declared.
  printf '%s\n' "$block" | script_lines | grep -qF 'await sleep(WRITE_INTERVAL_MS)'

  literal_const WRITE_INTERVAL_MS "$block"
  interval="$(printf '%s\n' "$block" | script_lines | grep -F 'const WRITE_INTERVAL_MS' | number_after_eq)"
  [ -n "$interval" ]
  [ "$interval" -gt 0 ]

  # 60000 / interval is the resulting writes-per-minute. Strictly under, for
  # the same reason the hourly cap below is: the event-driven writes share this
  # ceiling, so a sweep sized exactly to it leaves them nothing.
  rate="$(awk -v i="$interval" 'BEGIN { print int(60000 / i) }')"
  [ "$rate" -lt "$WRITES_PER_MINUTE" ]
}

@test "a single run cannot exceed the hourly ceiling" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # The cap has to gate the loop, not just exist as a constant.
  printf '%s\n' "$block" | script_lines | grep -qF 'if (writes >= MAX_WRITES_PER_RUN)'

  literal_const MAX_WRITES_PER_RUN "$block"
  cap="$(printf '%s\n' "$block" | script_lines | grep -F 'const MAX_WRITES_PER_RUN' | number_after_eq)"
  [ -n "$cap" ]
  [ "$cap" -gt 0 ]
  # Strictly under: a run sized exactly to the ceiling leaves no room for the
  # event-driven writes that share the same hourly budget.
  [ "$cap" -lt "$WRITES_PER_HOUR" ]
}

@test "a hung run cannot hold the queue for the default six hours" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # Every run of this workflow shares one concurrency group, so a call that
  # hangs blocks each issue filed behind it until the job times out. A capped
  # run paces roughly seven minutes, so the bound is generous and still far
  # under the 360-minute default.
  minutes="$(printf '%s\n' "$block" | code_lines | grep -F 'timeout-minutes:' | awk -F: 'NR == 1 { gsub(/[^0-9]/, "", $2); print $2 }')"
  [ -n "$minutes" ]
  [ "$minutes" -gt 0 ]
  [ "$minutes" -lt 60 ]
}

@test "the manual entry point defaults to dry-run" {
  # Bounded by indentation: nothing below the input is a key at this level, so
  # a stop condition matching the next sibling key would run to end of file and
  # call the rest of the workflow part of the block.
  block="$(code_lines < "$WF" | awk '
    /^      dry_run:$/ { inside = 1; next }
    inside && !/^        / { exit }
    inside')"
  [ -n "$block" ]

  # workflow_dispatch is the only way a human starts a repository-wide write,
  # and this is the whole of what stands between clicking Run and one. Every
  # other guard of that class is pinned above; this one is a single word away
  # from writing to every open issue on the first click, which is exactly the
  # shape the rest of this file exists to catch.
  default="$(printf '%s\n' "$block" | awk '/^[[:space:]]*default:/ { print $2; exit }')"
  [ "$default" = "true" ]

  # The declared default is what the form shows. It is deliberately not what
  # makes this safe: the derivation keys on the event and treats an absent or
  # unrecognised input as a dry run, which is why it is pinned separately.
  # Pinning only the default would leave `const dryRun = false;` green.
  wf="$(job_block triage "$WF")"
  printf '%s\n' "$wf" | script_lines | grep -qF "context.eventName !== 'schedule' &&"
  printf '%s\n' "$wf" | script_lines | grep -qF "String(context.payload.inputs?.dry_run ?? 'true') !== 'false'"
}

@test "a failed write neither aborts the sweep nor passes unnoticed" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # Without the catch, one issue transferred or deleted mid-sweep costs the run
  # the rest of the backlog and its summary line with it.
  printf '%s\n' "$block" | script_lines | grep -qF 'catch (err)'

  # A refusal is the other kind of failure and wants the opposite reaction: 403
  # is what both a lost `issues: write` and the secondary rate limit answer, 429
  # is the other throttle shape. The status is what separates them from the
  # benign 404 of a vanished issue, so the discrimination has to be on status.
  printf '%s\n' "$block" | script_lines | grep -qE "err\.status === 403.*err\.status === 429|err\.status === 429.*err\.status === 403"

  # And one refusal has to be enough to go red. Counting instead would make the
  # alarm dead in the regime the sweep lives in once the backlog is drained,
  # where a whole run writes to only a handful of issues.
  printf '%s\n' "$block" | script_lines | grep -qF 'if (refused > 0)'
  printf '%s\n' "$block" | script_lines | grep -qF 'core.setFailed'

  # And the run has to stop at the first refusal rather than carry on. A
  # refusal applies to every write left, so continuing repeats a rejected
  # request up to the cap, which is what gets an integration banned and what
  # burns the repository's hourly budget on a permission regression.
  # -A10 rather than -A5: the warning between the two is prose and gains a
  # line whenever it is reworded, which would fail this on a no-op edit.
  printf '%s\n' "$block" | script_lines | grep -A10 -F 'refused += 1;' | grep -qF 'break;'
}

@test "the sweep reports what it did in a job summary" {
  block="$(job_block triage "$WF")"
  [ -n "$block" ]

  # A sweep over the backlog logs a line per issue, so the summary is the only
  # place a reader gets the outcome without paging through it. `write()` is
  # async: unawaited, the promise can go unflushed and the step ends green with
  # an empty panel, which is indistinguishable from a run that did nothing.
  printf '%s\n' "$block" | script_lines | grep -qF 'await core.summary'
  printf '%s\n' "$block" | script_lines | grep -qF '.write();'

  # Every outcome a run can end on has to reach it, at zero as much as at ten:
  # reporting the writes and dropping the rest is what makes a run that stopped
  # early read as one that got to the end. The refusal itself is pinned by the
  # failure test above; what is pinned here is that the table accounts for it,
  # because otherwise the rows silently fail to sum to the listing.
  printf '%s\n' "$block" | script_lines | grep -qF '${deferred}'
  printf '%s\n' "$block" | script_lines | grep -qF '${unexamined}'
  printf '%s\n' "$block" | script_lines | grep -qF '${refused}'
}

# ── supply chain and token scope ─────────────────────────────────────────────

@test "every action is pinned to a digest" {
  # zizmor gates this tree-wide, but this file is the one that fails on the
  # workflow that has issues: write, so it pins its own.
  # Asserted as "every `uses:` is a digest" rather than "no tag matches a
  # tag-shaped pattern", because every such pattern has a gap: `@main` is
  # neither `@v7` nor `@1.2.3`, and a branch is the most mutable of the three.
  # Counting digests rather than naming the one action keeps a second, properly
  # pinned step from going red for no reason. A repo-local `./.github/actions/`
  # step would need an exemption here; it is not the same class of risk.
  total="$(code_lines < "$WF" | grep -cE '^[[:space:]]*(-[[:space:]]*)?uses:' || true)"
  pinned="$(code_lines < "$WF" | grep -cE '^[[:space:]]*(-[[:space:]]*)?uses:.*@[0-9a-f]{40}([[:space:]]|$)' || true)"
  [ "${total:-0}" -gt 0 ]
  [ "${total:-0}" -eq "${pinned:-0}" ]
}

@test "the token is read-only except for one job that writes issues" {
  # A job-level block replaces the top-level one rather than extending it, so
  # the two assertions below are the whole permission surface.
  code_lines < "$WF" | grep -qF 'contents: read'

  # Anchored to the YAML key, not matched as a substring. `code_lines` strips
  # `#` comments only, and `issues: write` also appears inside the script block
  # in a `//` comment and in a `core.setFailed` string, so a substring match is
  # satisfied by prose and stays green when the job's one scope is swapped for
  # `contents: write`. The count below does not catch that either: the swapped
  # line is still exactly one `write`.
  printf '%s\n' "$(job_block triage "$WF")" | code_lines |
    grep -qE '^[[:space:]]+issues:[[:space:]]*write[[:space:]]*(#.*)?$'

  # Nothing else is writable. This is what keeps a labeling workflow from
  # quietly acquiring contents: write or pull-requests: write later. The
  # trailing-comment form is matched too, because a permission gets added
  # with its justification on the same line, which is exactly the shape an
  # assertion anchored at `write$` would wave through.
  count="$(code_lines < "$WF" | grep -cE '^[[:space:]]+[a-z-]+:[[:space:]]*write[[:space:]]*(#.*)?$' || true)"
  [ "${count:-0}" -eq 1 ]
}

# ── interaction with the rest of the tree ────────────────────────────────────

@test "runs are serialised and never cancelled" {
  # A sweep works from a listing taken when it started. Letting a second run
  # overlap it lets the two read the same issue differently and leave it
  # holding two contradictory labels; cancelling one mid-sweep leaves the
  # backlog half labeled with a green-looking cancelled run.
  block="$(code_lines < "$WF" | awk '/^concurrency:/ { inside = 1; next } /^[a-z]/ { inside = 0 } inside')"
  [ -n "$block" ]
  printf '%s\n' "$block" | grep -qF 'cancel-in-progress: false'
}

@test "no accepted-signal label widens the stale exemption set" {
  [ -f "$STALE" ]

  # `triage/accepted` is itself on exempt-issue-labels, so awarding it for a
  # label that is NOT on that list quietly grants a permanent reprieve the
  # stale policy withheld — which is what treating every `priority/*` tier as
  # a signal did, `priority/backlog` being the tier stale.yaml leaves reapable.
  # An assignee is deliberately not covered here: it is not a label, and its
  # exemption is the one the workflow argues for out loud.
  exempt="$(awk '
    /^[[:space:]]*exempt-issue-labels:/ { inside = 1; next }
    /^[[:space:]]*[a-z-]+:/ { inside = 0 }
    inside' "$STALE")"
  [ -n "$exempt" ]

  # One entry per line, so the containment check below can be exact. The value
  # is a folded scalar: comma-separated entries wrapped across several indented
  # lines, each line carrying a trailing comma. Matching a signal label as a
  # substring of that block would pass one that is merely a prefix of an exempt
  # entry — `priority/important` against `priority/important-soon` — and that
  # prefix is a label the stale policy never exempted.
  exempt_labels="$(printf '%s\n' "$exempt" | tr ',' '\n' |
    sed 's/^[[:space:]]*//; s/[[:space:]]*$//' | grep -v '^$')"
  [ -n "$exempt_labels" ]

  signals="$(signal_labels)"

  # Non-vacuous: an empty list, or a classifier that stopped reading it, must
  # not let this pass by having nothing to check. One, not a higher floor —
  # a floor above the stated purpose fails on a legitimate removal that keeps
  # the list non-empty, which is a test edit with no contract behind it. This
  # branch shortened the list once already.
  count="$(printf '%s\n' "$signals" | grep -c . || true)"
  [ "${count:-0}" -ge 1 ]
  printf '%s\n' "$(job_block triage "$WF")" | script_lines | grep -qF 'ACCEPTED_SIGNAL_LABELS.includes(n)'

  for label in $signals; do
    printf '%s\n' "$exempt_labels" | grep -qxF "$label"
  done
}

@test "no lifecycle label sits in the accepted-signal list" {
  # Membership of stale.yaml's exempt list is necessary for a signal but not
  # sufficient, and the test above can only check the necessary half. A
  # `lifecycle/*` label states what the stale bot may do with an issue and
  # nothing about whether anybody reviewed it, so `lifecycle/frozen` is exempt
  # and still not a signal: freezing an untriaged issue to keep the bot off it
  # is exactly the case where reading it as accepted is false.
  #
  # `priority/backlog` is excluded for a different reason and needs no assertion
  # of its own, because it is not on the exempt list and the test above already
  # rejects it. That is a coincidence of the stale policy rather than a decision
  # taken here, and it stops covering the tier the moment somebody exempts it.
  signals="$(signal_labels)"

  # Guards the assertion below against passing on an empty list, which is what
  # a broken extraction and a compliant workflow would otherwise look like.
  [ -n "$signals" ]

  # Capture grep's status instead of writing `! grep -q ...`, which cannot fail
  # a test here. Two mechanisms, and the second is the one that carries this
  # case: `set -e` ignores a pipeline led by `!`, so such an assertion mid-body
  # never aborts, and `hack/cozytest.sh`'s translator appends `return 0` to
  # every line that is exactly `}`, so one written as the body's last statement
  # does not decide the status either. Under real bats the trailing form does
  # go red, which is the only position where the two runners differ at all: a
  # negated assertion with anything after it is green under both. So the rule
  # is not "bats is the honest runner", it is that no position is safe. Every
  # negative assertion in this file takes this shape, and a mutation ladder
  # proving otherwise has to run the runner CI uses.
  rc=0
  printf '%s\n' "$signals" | grep -q '^lifecycle/' || rc=$?
  [ "$rc" -ne 0 ]
}

@test "the sweep runs after the stale bot, not before it" {
  [ -f "$STALE" ]

  # Labeling bumps updated_at, which the stale bot reads as activity. Running
  # first would hand every issue it is about to label a fresh clock in the same
  # morning, so the sweep goes second and lets that day's stale pass complete
  # on the state it was scheduled against.
  # `-e`, because the pattern opens with a dash and grep would read it as one.
  sweep="$(code_lines < "$WF" | grep -F -e '- cron:' | cron_minutes)"
  stale="$(code_lines < "$STALE" | grep -F -e '- cron:' | cron_minutes)"
  [ -n "$sweep" ] && [ -n "$stale" ]
  [ "$sweep" -gt "$stale" ]
}

@test "both labels the workflow applies are declared in the repository" {
  [ -f "$LABELS" ]

  # addLabels creates a label that does not exist, with a default colour and no
  # description, so a typo here is invisible until someone notices two triage
  # labels with almost the same name.
  #
  # Whole-line match, for the same reason the exempt-list check above splits its
  # block one entry per line: a substring match accepts a truncation, and a
  # truncation is exactly what a typo looks like. `name: triage/needs-triag`
  # occurs inside `- name: triage/needs-triage`, so `grep -F` alone reports the
  # misspelled label as declared and the sweep then creates it on first write.
  # `-e`, because the whole-line pattern opens with the list dash and grep would
  # otherwise read it as an option, the same reason the cron check above needs it.
  #
  # The trailing comment is stripped before the match rather than allowed by the
  # pattern, so the check stays a fixed-string one: the label is interpolated,
  # and a regex would give whatever it contains meaning it does not have. A
  # declaration carrying its justification inline is the shape the permission
  # count above already tolerates, and there is no contract against it here.
  for label in $(script_lines < "$WF" | grep -E "const (NEEDS_TRIAGE|ACCEPTED) = " | sed "s/^[^']*'//; s/'.*$//"); do
    code_lines < "$LABELS" | sed 's/[[:space:]]*#.*$//; s/[[:space:]]*$//' |
      grep -qxF -e "- name: $label"
  done
}