#!/usr/bin/env bats # ----------------------------------------------------------------------------- # The ban on `!`-negated assertions in .bats files, and why the form that reads # most naturally is the one form that cannot fail. # # POSIX exempts a command whose return value is inverted with `!` from errexit, # and neither runner this repository uses recovers what that throws away. # # hack/cozytest.sh turns each @test into a shell function with `set -e` at the # top and `return 0` at the bottom, so a failing `! cmd` neither aborts the # function nor survives as its return value: it prints a pass whatever the # assertion found. The bats binary looks stricter and mostly is not. Its ERR # trap carries the same exemption -- `set -eE; trap ... ERR; ! true` runs the # trap not at all and exits zero -- so the only negated assertion it reports is # one standing LAST in the body, where the inverted status becomes the test # function's own. Move an `echo` below it and bats passes the test too. # # So this is not a disagreement between runners to be settled by picking the # stricter one. Measured on a fixture where each assertion must fail: bare # `! true`, a negated pipeline, a negated `[ ... ]`, one inside a `for` body, # and one following `;`, `&&` or `||` all pass under both, and the tail-position # case is the single shape the two answer differently. # # The damage is concentrated rather than spread, because `!` is the natural way # to spell "this must NOT appear", and an absence is precisely what no other # assertion in the test covers. A suite can hold a full set of them and be, on # that half of its contract, a suite that passes on any input. # # Write the absence so that it fails: # # if grep -q FORBIDDEN "$log"; then echo "FAIL: "; false; fi # # The `false` is what the runner sees. This is already the form used wherever # the trap was noticed one file at a time; the rule only makes it universal. # # No exception for `! cmd || { echo ...; exit 1; }`, which does work -- the `||` # reads the inverted status and the `exit` leaves the test. Admitting it means # the scan has to parse the tail of every statement to tell a guarded negation # from a bare one, over a set of working spellings that is open-ended, and a # scan that guesses wrong there is green on a real one. One rule, no tail. # # `if !`, `elif !`, `while !` and `until !` are untouched: errexit is not what # decides a condition, and inverting one is ordinary shell. `[ ! -f x ]` and # `!=` are not negated commands at all. # # The scan is lexical, with the limits that implies. A statement assembled at # runtime is invisible to it. In the other direction the form is counted where # no command exists -- inside a heredoc, or in a string a test compares against # -- which reports a file that holds nothing rather than passing one that does, # and is why the fixture writers below build the token from a separate argument. # # It knows the statement boundaries written into it and no others, so the test # below names the shapes it was measured against rather than claiming the # language. Those are the POSIX separators -- `;` `&` `&&` `||` `{` `(` the `)` # closing a case pattern, and the words `then` `do` `else`; a `time` or # `time -p` in front of the `!` is read through. Five known over-counts come # with that and are left alone, all erring loud: `( ! cmd )` does propagate its # status and is reported anyway; a `!` after a `&` used as a redirection rather # than a separator would be too, if one ever began a statement; # `x=$(true || ! false)` is reported although the `||` reads the inverted status # and the assignment takes it; a condition continued by a trailing `&&` rather # than a backslash -- `if foo &&` on one line, `! bar; then` on the next -- is # reported because the strip only recognises a condition it can see the end # of; and `time ! cmd` is reported although under cozytest on a dash /bin/sh, # which has no `time` keyword, it runs an external `time` that cannot find a # command named `!` and fails. Under bats, and under cozytest wherever /bin/sh # is bash, it is vacuous. Each costs a rewrite; the opposite error costs a # silent pass, which is why the list grows by naming rather than by narrowing. # # The verdict travels on stdout and not in an exit status. cozytest.sh appends # `return 0` before every closing brace at column zero, plain helper functions # included, so a helper whose answer is the status of its LAST command returns # success under this runner and its real status under bats. An explicit `return` # from inside the body still carries -- it runs first -- which is what makes the # masked case easy to write by accident and hard to see. A guard whose result # fell out of its final command would be green unconditionally: the same defect # it exists to ban, one level up. # ----------------------------------------------------------------------------- load test_helper # The directory to audit. Under the bats binary this file's own location; under # cozytest.sh, which sets no BATS_TEST_FILENAME, `$0` is the runner, and the # answer agrees only because the runner lives beside the files it runs. The # first test refuses to pass on an enumeration that missed this file, so a wrong # answer here fails instead of auditing nothing. BATS_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" RUNNER="$BATS_DIR/cozytest.sh" # Report each statement in file $1 whose first token is `!`, one line each, # naming the physical line the statement STARTS on. Whole-line comments are # blanked rather than dropped so the numbering survives them, and backslash # continuations are folded so a statement split across lines is read as one. cnb_scan() { awk -v name="${2:-$1}" ' { raw = $0 if (raw ~ /^[[:space:]]*#/) raw = "" if (buf == "") start = FNR buf = buf raw if (buf ~ /\\$/) { sub(/\\$/, " ", buf); next } line = buf; buf = "" if (line ~ /^[[:space:]]*(if|elif|while|until)[[:space:]]/) { if (match(line, /;[[:space:]]*(then|do)([[:space:]]|$)/)) line = substr(line, RSTART + RLENGTH) else next } # A keyword may be preceded only by whitespace or by `;` -- every other # separator in front of `then`, `do` or `else` is a shell syntax error, # checked with `sh -n` rather than recalled -- so those two characters are # the complete set and the class is closed rather than patched. The leading # space covers a keyword in column zero, which has nothing in front of it # at all. Miss either and the negation after the keyword never opens a # segment, which is silent, and this scan errs loud or not at all. No # apostrophes in here: the awk program is single-quoted. line = " " line n = split(line, seg, /(&&|&|\|\||;|\{|\(|\)|[[:space:];](then|do|else)[[:space:]])/) for (i = 1; i <= n; i++) { # In bash `time` and `time -p` prefix a pipeline and leave its `!` # exempt, and unlike the keywords they may follow any separator, so # they are peeled off the segment rather than split on. A `!` ending # its segment is one the split cut from a `(` that followed it with no # space: `!(cmd)`. s = seg[i] sub(/^[[:space:]]*time([[:space:]]+-p)?[[:space:]]+/, "", s) if (s ~ /^[[:space:]]*!([[:space:]]|$)/) { sub(/^[[:space:]]*/, "", s) print name ":" start ": " s break } } } ' "$1" } # Every offending statement under directory $1, and nothing when there are none. cnb_audit() { find "$1" -name '*.bats' | sort | while IFS= read -r _f; do [ -e "$_f" ] || continue cnb_scan "$_f" "${_f#"$1"/}" done return 0 } # Fixture writers. The token arrives as a separate argument on purpose: this # file is read by the scan it defines, and a fixture spelled out literally would # be reported as an offence in the guard's own source. cnb_new_fixture() { printf '%s\n' '#!/usr/bin/env bats' > "$1/subject.bats" return 0 } cnb_append() { _dir=$1; shift printf "$@" >> "$_dir/subject.bats" return 0 } @test "the audit resolves its root to this file's own directory" { files="$(find "$BATS_DIR" -name '*.bats' | wc -l)" [ "$files" -gt 1 ] # Reaching SOME file is not the claim -- reaching this one is, because a root # resolved from the runner instead of the suite would still enumerate a # plausible directory and report an audit of somewhere else as clean. find "$BATS_DIR" -name '*.bats' | grep -qF 'bats-no-negated-assert.bats' } @test "the audit recurses, and reports a subdirectory by its relative path" { # Through cnb_audit and not through a second copy of its find expression. An # assertion that retypes the command it means to check holds a copy: the # function can lose its recursion with the copy still passing, and then the # audit returns empty for a subdirectory and empty reads as a clean tree -- # this file's own subject, one level up. # # Recursion is load-bearing for a suite in a subdirectory: BATS_UNIT_FILES is # a non-recursive wildcard, so `make unit-tests` never runs one there, and # without recursion this audit would not read it either. hack/e2e-apps/ held # such suites until they moved to Chainsaw. The proof is a fixture rather than # a real directory, because none holds a suite today -- pinning against one # would turn this test red for a reason that has nothing to do with recursion. dir="$(mktemp -d)" mkdir -p "$dir/sub" cnb_new_fixture "$dir/sub" cnb_append "$dir/sub" '@test "a" {\n' cnb_append "$dir/sub" ' %s true\n' '!' cnb_append "$dir/sub" '}\n' report="$(cnb_audit "$dir")" # One report line and no more. An EMPTY report also satisfies this, because # `printf '%s\n' ""` emits a newline -- the grep below is what rejects it, # and that division of labour is the same in the continuation test. [ "$(printf '%s\n' "$report" | wc -l)" -eq 1 ] # Relative to the audited root, so hack/foo.bats and hack/e2e-apps/foo.bats # cannot report under the same name. printf '%s\n' "$report" | grep -q '^sub/subject.bats:3: ' rm -rf "$dir" } @test "no suite states an absence in a form that cannot fail" { report="$(cnb_audit "$BATS_DIR")" if [ -n "$report" ]; then echo 'A statement whose first token is `!` has its status exempted from errexit,' echo 'so the assertion below passes whatever it finds. Spell the absence as:' echo ' if cmd; then echo "FAIL: "; false; fi' printf '%s\n' "$report" false fi } @test "the scan reports a negated statement in every shape measured here" { # Every shape below was run under the runner and reported a pass while its # assertion must have failed, so each is a live hole the scan has to close; # the `time` shapes at the end are that on bash and not on dash. # The name claims what this fixture holds and nothing wider: a lexical scan # over shell cannot promise it has enumerated the language. dir="$(mktemp -d)" cnb_new_fixture "$dir" cnb_append "$dir" '@test "a" {\n' cnb_append "$dir" ' %s true\n' '!' cnb_append "$dir" ' %s printf x | grep -q x\n' '!' cnb_append "$dir" ' echo one; %s true\n' '!' cnb_append "$dir" ' true && %s true\n' '!' cnb_append "$dir" ' false || %s true\n' '!' cnb_append "$dir" ' for f in a b; do\n %s true\n done\n' '!' cnb_append "$dir" ' if true; then %s true; fi\n' '!' # A case branch: the `)` closing the pattern is a statement boundary like any # other, and `case` already appears inside test bodies and stubs in this tree. cnb_append "$dir" ' case "$v" in\n yes) %s true ;;\n esac\n' '!' # After a background `&`, which separates statements exactly as `;` does. cnb_append "$dir" ' true & %s true\n' '!' # A brace group takes the exemption with it: the group reports the inverted # status as its own, so the whole thing passes. A subshell does not, which is # why only this one is here. cnb_append "$dir" ' { %s true; }\n' '!' # The same keywords at column zero. Every other occurrence of `then`, `do` and # `else` in a test body is indented, which is what made the missing space # before them invisible: without it the keyword opens the segment and the # negation after it is never a segment's first token. All three are vacuous # under the runner, so the miss was silent -- the direction this file says it # will not err in. cnb_append "$dir" ' for f in a b\n' cnb_append "$dir" 'do %s true\n' '!' cnb_append "$dir" ' done\n' cnb_append "$dir" ' if false\n' cnb_append "$dir" 'then %s true\n' '!' cnb_append "$dir" 'else %s true\n' '!' cnb_append "$dir" ' fi\n' # The same keywords with no space after the `;`. Valid shell, vacuous under # the runner, and missed until the separator class admitted `;` in front of a # keyword as well as whitespace. cnb_append "$dir" ' for f in a;do %s true;done\n' '!' cnb_append "$dir" ' if x; then :;else %s true; fi\n' '!' # A negation with no space before its subshell, and one behind `time`, with # and without `-p` and with no space after the separator in front of it. The # first is vacuous under both runners on every /bin/sh measured, dash and # bash; the `time` ones under bats and under cozytest where /bin/sh is bash, # and on dash they are the over-count the header names. All three were missed # while the scan wanted whitespace after the `!` and read `time` as the # statement's first token. cnb_append "$dir" ' %s(true)\n' '!' cnb_append "$dir" ' time %s true\n' '!' cnb_append "$dir" ' true &&time -p %s true\n' '!' cnb_append "$dir" '}\n' found="$(cnb_scan "$dir/subject.bats" | wc -l)" [ "$found" -eq 18 ] rm -rf "$dir" } @test "the scan reports a subshell negation, and that stays an over-count" { # `( ! cmd )` is not a hole. The subshell takes the exemption and hands the # inverted status up as its own, and the caller is not exempt, so the # assertion does fail. The scan reports it regardless, because admitting the # shape means telling a guarded negation from a bare one by reading what # follows, over a set of working spellings with no end to it. # # This and `!(true)` in the shapes test are the two fixtures that reach `(` # in the separator class. Every other shape carries a second boundary on its # own line and would still be reported with `(` dropped from that class. dir="$(mktemp -d)" cnb_new_fixture "$dir" cnb_append "$dir" '@test "a" {\n' cnb_append "$dir" ' ( %s true )\n' '!' cnb_append "$dir" '}\n' report="$(cnb_scan "$dir/subject.bats")" [ "$(printf '%s\n' "$report" | wc -l)" -eq 1 ] printf '%s\n' "$report" | grep -q ':3: ' rm -rf "$dir" # The other half of the word "over-count": the shape has to actually work, # or what is pinned above is a second hole wearing the name of a false # positive. Assembled rather than written out, for the reason the header # gives. prog="$(printf 'set -e; ( %s true ); echo REACHED' '!')" [ -z "$(sh -c "$prog" 2>&1)" ] } @test "the scan folds a continuation and names the line the statement opens on" { dir="$(mktemp -d)" cnb_new_fixture "$dir" cnb_append "$dir" '@test "a" {\n' # The negation has to land on the CONTINUATION line, not the line that opens # it. Put it on the opening line and the report says line 3 whether the fold # ran or not, so the assertion below passes with the folding deleted -- which # is the shape this whole suite exists to ban. Here folded reports 3 and # unfolded reports 4, so removing the fold turns the test red. # # `\\\n` and not `\\\\\n`: printf collapses each pair, so the doubled form # emits two backslashes -- an escaped literal, which is an ordinary argument # rather than a continuation. Check the bytes with `od -c` when editing this. cnb_append "$dir" ' echo start \\\n' cnb_append "$dir" ' ; %s grep -q x "$file"\n' '!' cnb_append "$dir" '}\n' report="$(cnb_scan "$dir/subject.bats")" # One report line and no more. An empty report satisfies this too -- one # newline out of printf -- so the grep below is what rules empty out, not # this line. [ "$(printf '%s\n' "$report" | wc -l)" -eq 1 ] printf '%s\n' "$report" | grep -q ':3: ' rm -rf "$dir" } @test "the scan leaves an inverted condition and an inverted test operator alone" { dir="$(mktemp -d)" cnb_new_fixture "$dir" cnb_append "$dir" '@test "a" {\n' cnb_append "$dir" ' if %s command -v act >/dev/null; then echo none; fi\n' '!' cnb_append "$dir" ' if %s foo || %s bar; then echo none; fi\n' '!' '!' cnb_append "$dir" ' while %s ready; do sleep 1; done\n' '!' cnb_append "$dir" ' until %s done_yet; do sleep 1; done\n' '!' # The two lines above cannot tell whether `while` and `until` are in the # condition-strip: splitting on `;` and ` do ` already leaves `!` in the # middle of segment 1, so they report nothing either way. These two put the # negation where only the strip can save it -- drop either keyword from the # alternation and the line becomes a report. cnb_append "$dir" ' while foo && %s bar; do sleep 1; done\n' '!' cnb_append "$dir" ' until foo && %s bar; do sleep 1; done\n' '!' cnb_append "$dir" ' [ %s -f /nowhere ]\n' '!' cnb_append "$dir" ' [ "$a" %s= "$b" ]\n' '!' cnb_append "$dir" ' # a comment naming %s grep -q x file\n' '!' cnb_append "$dir" '}\n' found="$(cnb_scan "$dir/subject.bats" | wc -l)" [ "$found" -eq 0 ] # An empty report is what a scan that reads nothing returns too, so the same # fixture has to produce one when the offence is present. cnb_append "$dir" '@test "b" {\n' cnb_append "$dir" ' %s true\n' '!' cnb_append "$dir" '}\n' found="$(cnb_scan "$dir/subject.bats" | wc -l)" [ "$found" -eq 1 ] rm -rf "$dir" } @test "a negated assertion still passes under this runner whatever it finds" { # The premise the ban rests on, pinned rather than described: the day the # runner starts propagating an inverted status, this test fails and the rule # above can be reconsidered instead of outliving its reason. dir="$(mktemp -d)" { printf '@test "an assertion that must not hold" {\n' printf ' %s true\n' '!' printf '}\n' } > "$dir/pin.bats" rc=0 "$RUNNER" "$dir/pin.bats" > "$dir/out" 2>&1 || rc=$? # Both halves: the runner reported success, AND a test actually ran to produce # it. Without the second, a fixture the runner refused to parse would satisfy # the first and pin nothing. [ "$rc" -eq 0 ] grep -qF 'Test OK' "$dir/out" rm -rf "$dir" } @test "the form this ban prescribes does fail, under both runners" { # The mirror of the test above, and the half more worth pinning. Every absence # converted in this tree rests on `false` inside a `then` body reaching the # runner as a failure. If that ever stopped -- a stray `set +e`, a change to # the `return 0` insertion in cozytest.sh -- the test above would still pass, # because a negated assertion would still be vacuous, while every converted # absence went silently green. The ban would then be prescribing a form that # checks nothing, which is the defect it exists to remove. # # Under both runners because their disagreement is the reason this class # exists at all: a remedy that only propagates under one of them is not a # remedy for the suites CI runs. dir="$(mktemp -d)" { printf '@test "an assertion that must fail" {\n' printf ' if true; then echo "FAIL: the prescribed form fired"; false; fi\n' printf ' echo PAST-THE-FORM\n' printf '}\n' } > "$dir/pin.bats" rc=0 "$RUNNER" "$dir/pin.bats" > "$dir/out" 2>&1 || rc=$? [ "$rc" -ne 0 ] # It has to fail AT the form. A run that died earlier -- an unparsable # fixture, a missing runner -- would satisfy the status check and pin nothing. grep -qF 'the prescribed form fired' "$dir/out" if grep -qF 'PAST-THE-FORM' "$dir/out"; then echo "FAIL: execution continued past the prescribed form" false fi if command -v bats >/dev/null 2>&1; then rc=0 bats "$dir/pin.bats" > "$dir/bats-out" 2>&1 || rc=$? [ "$rc" -ne 0 ] grep -qE '^not ok ' "$dir/bats-out" else echo "bats binary absent — the prescribed form NOT verified under it in this run." >&2 fi rm -rf "$dir" } @test "neither errexit nor the ERR trap sees an inverted status" { # The half of the premise that says switching runners does not lift the ban. # Pinned at the shell rather than through the bats binary, because this is # where the exemption lives -- bats builds its reporting on exactly these two # mechanisms, so a suite host without bats installed can still hold the claim # honest. A shell without `bash` cannot run bats either, which is the only # case this skips. # `skip` is a bats builtin that cozytest.sh does not provide, so the note goes # out the way the other suites here report an unmet dependency. if ! command -v bash >/dev/null 2>&1; then echo "bash unavailable — the errexit and ERR-trap exemptions NOT verified in this run." >&2 return 0 fi # Assembled rather than written out, for the reason the header gives: spelled # literally, this string is a statement the scan reports against its own file. prog="$(printf 'set -eE; trap "echo TRAP-RAN" ERR; %s true; echo REACHED' '!')" out="$(bash -c "$prog" 2>&1)" # Reached the line below the failing assertion, and the trap never fired: # either alone would leave the other half unpinned. [ "$out" = "REACHED" ] }