test-run.sh

#!/bin/bash
## The high road: testmarktest_rule_basic, testmarktest_provide, and testmarktest_run.
## Each case writes a doc and a test script, then runs the script in a fresh bash, from another directory.
## These assert in plain bash, not through the runner itself, so a broken runner can't pass its own tests.
set -euo pipefail
cd "$(dirname "$0")"
# shellcheck source=tests/lib.sh
source ./lib.sh

unset TESTMARK_REGEN
mkdir t

## Writes t/doc.md and t/test.sh, runs the script from $work, and checks what it printed and its status.
## The script gets a line sourcing the library first.  $work in the output shows as WORK.
##
## - @param $1 description
## - @param $2 t/doc.md's content
## - @param $3 the rest of t/test.sh
## - @param $4 the expected stdout, then a line `--- stderr`, then the expected stderr, then `--- status N`
run_case() {
	printf '%s' "$2" >t/doc.md
	printf 'source %q\n%s\n' "$ROOT/lib/testmark.bash" "$3" >t/test.sh
	local rc=0 got
	"$BASH" t/test.sh >"$work/out" 2>"$work/err" </dev/null || rc=$?
	got="$(<"$work/out")"$'\n'"--- stderr"$'\n'"$(<"$work/err")"$'\n'"--- status $rc"
	got=${got//$work/WORK}
	expect_eq "$1" "$got" "$4"
}

## Checks t/doc.md's bytes, as `bytes` quotes them.
##
## - @param $1 description
## - @param $2 the expected document, as a printf format
doc_is() {
	expect_eq "$1" "$(bytes t/doc.md)" "$(esc "$(printf -- "$2" && printf x)" | sed 's/x"$/"/')"
}

## Prints a doc with one case, `a`: `a/in` holds "hello", and `a/output` holds $1, which must be empty or end in a newline.
one() {
	printf '[testmark]:# (a/in)\n```\nhello\n```\n\n[testmark]:# (a/output)\n```\n%s```\n' "$1"
}

# --- passing, failing, and what the output stream holds ---

run_case "a match passes" "$(one $'hello\n')" '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_cat)
--- stderr

--- status 0"

run_case "a mismatch fails, with a diff" "$(one $'bye\n')" '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md' "FAIL  doc.md: a/in (exercise_cat)
--- stderr
testmark: testmarktest_run: hunk \"a/output\": output doesn't match; set TESTMARK_REGEN=1 to accept it
--- WORK/t/doc.md: a/output (expected)
+++ WORK/t/doc.md: a/output (actual)
@@ -1 +1 @@
-bye
+hello
--- status 1"

run_case "stderr is in the stream, in order" "$(one $'out\nerr\nout\n')" '
exercise_both() { echo out; echo err >&2; echo out; }
testmarktest_rule_basic "*/in" exercise_both
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_both)
--- stderr

--- status 0"

run_case "a nonzero exit is a line of output" "$(one $'hello\nexit status: 3\n')" '
exercise_fail() { cat; return 3; }
testmarktest_rule_basic "*/in" exercise_fail
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_fail)
--- stderr

--- status 0"

run_case "no final newline gets a marker" "$(one $'x\n\\ No newline at end of output\n')" '
exercise_x() { printf x; }
testmarktest_rule_basic "*/in" exercise_x
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_x)
--- stderr

--- status 0"

run_case "the marker, then the exit status" "$(one $'x\n\\ No newline at end of output\nexit status: 1\n')" '
exercise_x() { printf x; false; }
testmarktest_rule_basic "*/in" exercise_x
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_x)
--- stderr

--- status 0"

run_case "empty output matches an empty hunk" "$(one '')" '
exercise_quiet() { :; }
testmarktest_rule_basic "*/in" exercise_quiet
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_quiet)
--- stderr

--- status 0"

run_case "CRLF fails the case" "$(one '')" '
exercise_crlf() { printf "a\r\n"; }
testmarktest_rule_basic "*/in" exercise_crlf
testmarktest_run doc.md' "FAIL  doc.md: a/in (exercise_crlf)
--- stderr
testmark: testmarktest_run: hunk \"a/output\": output must be empty or end in a newline, and contain no CRLF; normalize it before checking
--- status 1"

run_case "a NUL byte fails the case" "$(one '')" '
exercise_nul() { printf "a\0b\n"; }
testmarktest_rule_basic "*/in" exercise_nul
testmarktest_run doc.md' "FAIL  doc.md: a/in (exercise_nul)
--- stderr
testmark: testmarktest_run: hunk \"a/in\": output contains a NUL byte
--- status 1"

# --- the case's process ---

run_case "stdin is the trigger's body, exactly" $'[testmark]:# (a/in)\n```\n  two\\\n\n```\n\n[testmark]:# (a/output)\n```\n  two\\\n\n|\n```\n\n[testmark]:# (b/in)\n```\n```\n\n[testmark]:# (b/output)\n```\n|\n```\n' '
exercise_bar() { cat; echo "|"; }
testmarktest_rule_basic "*/in" exercise_bar
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_bar)
PASS  doc.md: b/in (exercise_bar)
--- stderr

--- status 0"

run_case "set -e works in a case, even with the run under if and ||" "$(one $'exit status: 1\n')" '
exercise_e() { false; echo unreached; }
testmarktest_rule_basic "*/in" exercise_e
if testmarktest_run doc.md; then echo ran; fi
testmarktest_run doc.md || echo failed' "PASS  doc.md: a/in (exercise_e)
ran
PASS  doc.md: a/in (exercise_e)
--- stderr

--- status 0"

run_case "pipefail is on in a case" "$(one $'exit status: 1\n')" '
exercise_p() { false | true; echo unreached; }
testmarktest_rule_basic "*/in" exercise_p
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_p)
--- stderr

--- status 0"

run_case "nounset is on in a case" "$(one $'/dev/fd/63: line 4: nope: unbound variable\nexit status: 1\n')" '
exercise_u() { echo "$nope"; }
testmarktest_rule_basic "*/in" exercise_u
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_u)
--- stderr

--- status 0"

run_case "the caller's set -e doesn't stop a run partway" '[testmark]:# (a/in)
```
```

[testmark]:# (a/output)
```
wrong
```

[testmark]:# (b/in)
```
```

[testmark]:# (b/output)
```
```' '
set -euo pipefail
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md
echo unreached' "FAIL  doc.md: a/in (exercise_cat)
PASS  doc.md: b/in (exercise_cat)
--- stderr
testmark: testmarktest_run: hunk \"a/output\": output doesn't match; set TESTMARK_REGEN=1 to accept it
--- WORK/t/doc.md: a/output (expected)
+++ WORK/t/doc.md: a/output (actual)
@@ -1 +0,0 @@
-wrong
--- status 1"

run_case "the case runs in the script's dir; docs are found there" "$(one "$work/t"$'\n')" '
exercise_pwd() { pwd; }
testmarktest_rule_basic "*/in" exercise_pwd
cd /
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_pwd)
--- stderr

--- status 0"

run_case "exported variables cross; others don't" "$(one $'yes\nunset\n')" '
export EXPORTED=yes
NOT_EXPORTED=no
exercise_env() { echo "$EXPORTED"; echo "${NOT_EXPORTED-unset}"; }
testmarktest_rule_basic "*/in" exercise_env
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_env)
--- stderr

--- status 0"

run_case "the caller's exported LC_ALL crosses, not the library's" "$(one $'C.UTF-8\n')" '
export LC_ALL=C.UTF-8
exercise_lc() { echo "$LC_ALL"; }
testmarktest_rule_basic "*/in" exercise_lc
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_lc)
--- stderr

--- status 0"

run_case "an unset LC_ALL stays unset in the case" "$(one $'unset\n')" '
unset LC_ALL
exercise_lc() { echo "${LC_ALL-unset}"; }
testmarktest_rule_basic "*/in" exercise_lc
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_lc)
--- stderr

--- status 0"

run_case "other functions don't cross, the library included" "$(one $'/dev/fd/63: line 5: helper: command not found\nexit status: 127\n')" '
helper() { echo helped; }
exercise_h() { declare -F | grep testmark || :; helper; }
testmarktest_rule_basic "*/in" exercise_h
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_h)
--- stderr

--- status 0"

run_case "provided functions cross" "$(one $'helped\n')" '
helper() { echo helped; }
exercise_h() { helper; }
testmarktest_provide helper
testmarktest_rule_basic "*/in" exercise_h
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_h)
--- stderr

--- status 0"

# --- regen ---

run_case "regen writes the output" "$(one $'bye\n')" '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
TESTMARK_REGEN=1 testmarktest_run doc.md && testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_cat)
PASS  doc.md: a/in (exercise_cat)
--- stderr

--- status 0"
expect_eq "regen wrote the doc" "$(<t/doc.md)" "$(one $'hello\n')"

run_case "regen writes every doc, and only what changed" "$(one $'hello\n')" '
printf "%s\n" "[testmark]:# (b/in)" "\`\`\`" "x" "\`\`\`" "" "[testmark]:# (b/output)" "\`\`\`text" "\`\`\`" >"${BASH_SOURCE%/*}/two.md"
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
TESTMARK_REGEN=1 testmarktest_run doc.md two.md
cat "${BASH_SOURCE%/*}/two.md"' "PASS  doc.md: a/in (exercise_cat)
PASS  two.md: b/in (exercise_cat)
[testmark]:# (b/in)
\`\`\`
x
\`\`\`

[testmark]:# (b/output)
\`\`\`text
x
\`\`\`
--- stderr

--- status 0"
expect_eq "regen left an unchanged doc as it was" "$(<t/doc.md)" "$(one $'hello\n')"

# --- what makes a run fail, besides a case ---

run_case "a missing output hunk fails the case" '[testmark]:# (a/in)
```
```' '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md
TESTMARK_REGEN=1 testmarktest_run doc.md' "FAIL  doc.md: a/in (exercise_cat)
FAIL  doc.md: a/in (exercise_cat)
--- stderr
testmark: testmarktest_run: no hunk named \"a/output\"
testmark: testmarktest_run: no hunk named \"a/output\"
--- status 1"

run_case "a trigger without a / fails" '[testmark]:# (in)
```
```' '
exercise_cat() { cat; }
testmarktest_rule_basic "*in" exercise_cat
testmarktest_run doc.md' "FAIL  doc.md: in (exercise_cat)
--- stderr
testmark: testmarktest_run: hunk \"in\": has no \"/\", so it names no case
--- status 1"

run_case "a rule matching nothing fails the run, after every case runs" "$(one $'hello\n')" '
exercise_cat() { cat; }
testmarktest_rule_basic "*/inn" exercise_cat
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md' "PASS  doc.md: a/in (exercise_cat)
--- stderr
testmark: rule \"*/inn\" exercise_cat: matched no hunk
--- status 1"

run_case "a doc with no match fails the run" "$(one $'hello\n')" '
printf "%s\n" "[testmark]:# (b/script)" "\`\`\`" "\`\`\`" >"${BASH_SOURCE%/*}/two.md"
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run two.md doc.md' "PASS  doc.md: a/in (exercise_cat)
--- stderr
testmark: two.md: no hunk matches any rule
--- status 1"

run_case "a doc that doesn't parse fails the run; the others still run" "$(one $'hello\n')" '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run nope.md doc.md' "FAIL  nope.md
PASS  doc.md: a/in (exercise_cat)
--- stderr
testmark: WORK/t/nope.md: not a readable file
--- status 1"

run_case "a run with no rules is an error" "$(one '')" '
testmarktest_run doc.md' "
--- stderr
testmark: testmarktest_run: no rules; make them with testmarktest_rule_basic first
--- status 1"

# --- several rules ---

run_case "rules pick out their own hunks, in document order" '[testmark]:# (one/script)
```
echo one
```

[testmark]:# (one/output)
```
one
```

[testmark]:# (two/in)
```
two
```

[testmark]:# (two/output)
```
two
```

[testmark]:# (three/script)
```
echo three
```

[testmark]:# (three/output)
```
three
```' '
exercise_script() { bash; }
exercise_cat() { cat; }
testmarktest_rule_basic "*/script" exercise_script
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md' "PASS  doc.md: one/script (exercise_script)
PASS  doc.md: two/in (exercise_cat)
PASS  doc.md: three/script (exercise_script)
--- stderr

--- status 0"

run_case "a glob's * matches /; the case is all but the last segment" '[testmark]:# (a/b/c/in)
```
deep
```

[testmark]:# (a/b/c/output)
```
deep
```' '
exercise_cat() { cat; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_run doc.md' "PASS  doc.md: a/b/c/in (exercise_cat)
--- stderr

--- status 0"

# --- arguments ---

defined() { :; }
expect_fail "rule: no arguments" 'testmark: usage: testmarktest_rule_basic GLOB FN' testmarktest_rule_basic
expect_fail "rule: one argument" 'testmark: usage: testmarktest_rule_basic GLOB FN' testmarktest_rule_basic '*/in'
expect_fail "rule: three arguments" 'testmark: usage: testmarktest_rule_basic GLOB FN' testmarktest_rule_basic '*/in' defined output
expect_fail "rule: an empty glob" 'testmark: testmarktest_rule_basic: the glob is empty' testmarktest_rule_basic '' defined
expect_fail "rule: not a function" 'testmark: testmarktest_rule_basic: "cat" is not a defined function' testmarktest_rule_basic '*/in' cat
expect_fail "provide: no arguments" 'testmark: usage: testmarktest_provide FN...' testmarktest_provide
expect_fail "provide: not a function" 'testmark: testmarktest_provide: "nope" is not a defined function' testmarktest_provide defined nope
expect_fail "run: no arguments" 'testmark: usage: testmarktest_run DOC...' testmarktest_run
expect_eq "refused calls leave no rules" "${#_testmarktest_fns[@]} ${#_testmarktest_provided[@]}" "0 0"
got=$("$BASH" -c 'source "$1"; f() { :; }; testmarktest_rule_basic "*" f; testmarktest_run doc.md' _ "$ROOT/lib/testmark.bash" 2>&1) || got+=" (status $?)"
expect_eq "run: not from a script file" "$got" 'testmark: testmarktest_run: no script file to find the docs beside; run it from a script (status 1)'

exit "$RESULT"