test-run-full.sh
#!/bin/bash
## The full road: testmarktest_rule, and the testcase_* functions its case functions call.
## Each case writes a doc and a test script, then runs the script in a fresh bash, from another directory.
## These assert in plain bash, not through the runner itself, so a broken runner can't pass its own tests.
set -euo pipefail
cd "$(dirname "$0")"
# shellcheck source=tests/lib.sh
source ./lib.sh
unset TESTMARK_REGEN TESTMARK_RAW_ERRORS
## Prints a doc with one case, `a`: `a/in` holds "hello", `a/out` holds $1, and `a/side` holds $2; both empty or ending in a newline.
## A hunk outside the case, `shared/x`, holds "shared".
one() {
printf '[testmark]:# (shared/x)\n```\nshared\n```\n\n[testmark]:# (a/in)\n```\nhello\n```\n\n[testmark]:# (a/out)\n```\n%s```\n\n[testmark]:# (a/side)\n```\n%s```\n' "$1" "$2"
}
# --- reading hunks, and outputs ---
run_case "outputs are checked against the case's hunks" "$(one $'hello\n' $'shared\n')" '
exercise_x() {
testcase_hunk in | testcase_output out
testcase_doc_hunk shared/x | testcase_output side
}
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
run_case "a mismatch fails, with a diff" "$(one $'bye\n' '')" '
exercise_x() { testcase_hunk in | testcase_output out; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md:11: hunk \"a/out\": output doesn't match
--- WORK/t/doc.md: a/out (expected)
+++ WORK/t/doc.md: a/out (actual)
@@ -1 +1 @@
-bye
+hello
--- status 1"
run_case "an output not given isn't checked" "$(one $'never\n' $'side\n')" '
exercise_x() { echo side | testcase_output side; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
run_case "outputs are exact: empty, blank lines, no final newline, odd bytes" "$(one '' $'\n\n \\\\ $(x) `y` \xff\n')" '
exercise_x() {
printf "" | testcase_output out
printf "\n\n \\\\\\\\ \$(x) \`y\` \xff" | testcase_output side
}
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
run_case "hunks read back exactly, whatever their names and bytes" '[testmark]:# (a/in)
```
```
[testmark]:# (a/$(echo)`echo`["x"]'"'"'y)
```
\\ $(x) `y` '$'\xff\r''
```
[testmark]:# (a/out)
```
\\ $(x) `y` '$'\xff''
```' '
exercise_x() { testcase_hunk "\$(echo)\`echo\`[\"x\"]'"'"'y" | testcase_output out; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
# --- what the case may not print ---
run_case "printing outside testcase_output fails the case; regen can't fix it" "$(one '' '')" '
exercise_x() { echo stray; echo err >&2; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md
TESTMARK_REGEN=1 testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it
--- WORK/t/doc.md: a/in (expected nothing)
+++ WORK/t/doc.md: a/in (actual)
@@ -0,0 +1,2 @@
+stray
+err
testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it
--- WORK/t/doc.md: a/in (expected nothing)
+++ WORK/t/doc.md: a/in (actual)
@@ -0,0 +1,2 @@
+stray
+err
--- status 1"
expect_eq "and the doc is unchanged" "$(<t/doc.md)" "$(one '' '')"
run_case "a nonzero exit fails the case, and outputs given before it are still checked" "$(one $'hello\n' '')" '
exercise_x() { testcase_hunk in | testcase_output out; false; echo unreached; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it
--- WORK/t/doc.md: a/in (expected nothing)
+++ WORK/t/doc.md: a/in (actual)
@@ -0,0 +1 @@
+exit status: 1
--- status 1"
run_case "set -e works in a case, even with the run under ||" "$(one '' '')" '
exercise_x() { false; echo unreached | testcase_output out; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md 2>/dev/null || echo failed' "FAIL doc.md: a/in (exercise_x)
failed
--- stderr
--- status 0"
run_case "a failing tool in a pipe into testcase_output fails the case" "$(one $'part\n' '')" '
exercise_x() { { echo part; false; } | testcase_output out; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md 2>&1 | grep -F exit' "+exit status: 1
--- stderr
--- status 0"
run_case "stdin is empty" "$(one $'[]\n' '')" '
exercise_x() { echo "[$(cat)]" | testcase_output out; }
testmarktest_rule "*/in" exercise_x
echo leaked | testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
# --- the testcase_* functions refusing ---
run_case "a missing hunk fails the case, naming it" "$(one '' '')" '
exercise_x() { testcase_hunk nope; }
exercise_y() { testcase_doc_hunk in; }
testmarktest_rule "*/in" exercise_x
testmarktest_rule "*/in" exercise_y
testmarktest_run doc.md 2>&1 | grep -E "^(\+[^+]|FAIL)"' "+testcase_hunk: no hunk named \"a/nope\"
+exit status: 1
FAIL doc.md: a/in (exercise_x)
+testcase_doc_hunk: no hunk named \"in\"
+exit status: 1
FAIL doc.md: a/in (exercise_y)
--- stderr
--- status 0"
run_case "testcase_* functions refuse bad arguments" "$(one '' '')" '
exercise_x() {
testcase_hunk || :
testcase_doc_hunk a b || :
testcase_output </dev/null || :
testcase_output /a/out </dev/null || :
testcase_output "a b" </dev/null || :
testcase_output "" </dev/null || :
printf "a\0b" | testcase_output out || :
testcase_withtmpdir x || :
}
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md 2>&1 | grep "^+[^+]"' '+usage: testcase_hunk NAME
+usage: testcase_doc_hunk NAME
+usage: testcase_output NAME <OUTPUT
+testcase_output: "/a/out" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace
+testcase_output: "a b" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace
+testcase_output: "" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace
+testcase_output: out: output contains a NUL byte
+usage: testcase_withtmpdir
--- stderr
--- status 0'
run_case "a NUL byte printed outside testcase_output fails the case" "$(one '' '')" '
exercise_x() { printf "a\0b\n"; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md 2>&1 | grep "^+[^+]"' '+a
+testmark: the output contains a NUL byte
--- stderr
--- status 0'
run_case "an output with no hunk fails the case" "$(one '' '')" '
exercise_x() { echo x | testcase_output nope; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md: no hunk named \"a/nope\"
--- status 1"
run_case "one output given twice, differently, fails the case" "$(one $'x\n' '')" '
exercise_x() { echo x | testcase_output out; echo y | testcase_output out; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md:11: hunk \"a/out\": checked twice, with different output
--- status 1"
run_case "a write to fd 3 garbles the report, and fails the case" "$(one '' '')" '
exercise_x() { echo oops >&3; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x)
--- stderr
testmark: WORK/t/doc.md:6: hunk \"a/in\": the case's report to the runner was garbled or cut short; nothing else in a case may write to fd 3
--- status 1"
# --- the working directory ---
run_case "testcase_withtmpdir moves the case into a new dir, removed after; TESTCASE_DIR still names the script's" "$(one $'in tmp: yes\nwrote: hi\n' '')" '
exercise_x() {
testcase_withtmpdir
echo hi >f
{ [[ $PWD != "$TESTCASE_DIR" ]] && echo "in tmp: yes"; echo "wrote: $(<f)"; } | testcase_output out
echo "$PWD" >"$TESTCASE_DIR/tmpdir"
}
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md
[[ ! -e $(<"${BASH_SOURCE%/*}/tmpdir") ]] && echo removed' "PASS doc.md: a/in (exercise_x)
removed
--- stderr
--- status 0"
# --- regen, and mixing the roads ---
run_case "regen writes every output given" "$(one '' '')" '
exercise_x() { testcase_hunk in | testcase_output out; echo s | testcase_output side; }
testmarktest_rule "*/in" exercise_x
TESTMARK_REGEN=1 testmarktest_run doc.md && testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x)
PASS doc.md: a/in (exercise_x)
--- stderr
--- status 0"
expect_eq "regen wrote the doc" "$(<t/doc.md)" "$(one $'hello\n' $'s\n')"
run_case "both roads in one run" '[testmark]:# (a/in)
```
hello
```
[testmark]:# (a/output)
```
hello
```
[testmark]:# (b/script)
```
```
[testmark]:# (b/out)
```
hello
```' '
exercise_cat() { cat; }
exercise_full() { testcase_doc_hunk a/in | testcase_output out; }
testmarktest_rule_basic "*/in" exercise_cat
testmarktest_rule "*/script" exercise_full
testmarktest_run doc.md' "PASS doc.md: a/in (exercise_cat)
PASS doc.md: b/script (exercise_full)
--- stderr
--- status 0"
run_case "bash's errors are relabeled, in outputs and outside them" "$(one $'exercise_x: nope_a: command not found\n' '')" '
exercise_x() { { nope_a || :; } 2>&1 | testcase_output out; nope_b; }
testmarktest_rule "*/in" exercise_x
testmarktest_run doc.md 2>&1 | grep -E "^(\+[^+]|FAIL)"' "+exercise_x: nope_b: command not found
+exit status: 127
FAIL doc.md: a/in (exercise_x)
--- stderr
--- status 0"
run_case "the testcase_* functions don't exist outside a case" "$(one '' '')" '
declare -F | grep -c testcase || :' "0
--- stderr
--- status 0"
# --- arguments ---
defined() { :; }
expect_fail "rule: one argument" 'testmark: usage: testmarktest_rule GLOB FN' testmarktest_rule '*/in'
expect_fail "rule: three arguments" 'testmark: usage: testmarktest_rule GLOB FN' testmarktest_rule '*/in' defined x
expect_fail "rule: an empty glob" 'testmark: testmarktest_rule: the glob is empty' testmarktest_rule '' defined
expect_fail "rule: not a function" 'testmark: testmarktest_rule: "cat" is not a defined function' testmarktest_rule '*/in' cat
expect_eq "refused calls leave no rules" "${#_testmarktest_fns[@]}" "0"
exit "$RESULT"
tests/test-run-full.sh · 10.2 KiB · 5cf95dcbac