run: testmarktest_rule, whose case functions read any hunk and give any outputs, with testcase_* functions.
commit e556796862d58841d101bb5920d6a718b332ec17 · parent f110c869d0
· browse this tree
Ximianth, 2026-10-25 12:04:34 UTC
testcase_read_myhunk and testcase_read_anyhunk print a hunk in the case, or anywhere in the doc;
testcase_output NAME checks its stdin against the case's hunk NAME;
testcase_withtmpdir moves the case into a temp dir, removed when it ends.
Anything else the case prints, or a nonzero exit, fails it: there's no hunk to put it in.
The testcase_* functions live in lib/testmark/testcase.bash, which the caller never sources:
its text goes into each case's bash, like the case's function.
How data moves, and why:
- The doc's bodies reach the case as raw bytes on fds 8 and 9, read with $(<...), with an index of offsets.
Bash reads script text from a pipe a byte at a time, and parses it: a 120 KB doc as declare -p in the script cost 38 ms per case; this costs about 3.
- Outputs come back as length-prefixed frames on the case's stdout (fd 3 inside).
The case function runs as { FN; } 2>&1 | _testcase_stream, which frames the leftover output once every writer has exited,
so frames never interleave, and FN's set -e holds (it's a new bash process, not a tested context).
- Frame lengths are checked as digits before any arithmetic, and a garbled report fails the case.
- read -N 0 reads to the end on bash 4.4, so empty frames skip the read.
Each behaviour was mutated out once and some test failed; tests pass on 4.4, 5.0, 5.1, 5.3.
diff --git a/lib/testmark.bash b/lib/testmark.bash index d4d7590..470ca26 100644 --- a/lib/testmark.bash +++ b/lib/testmark.bash @@ -115,6 +115,24 @@ testmark_check() { _testmark_call _testmark_check "$@"; } ## - @param $2 FN: the name of a defined function. testmarktest_rule_basic() { _testmark_call _testmarktest_rule_basic "$@"; } +## Makes every hunk whose name matches GLOB a case, to be exercised by FN, which reads hunks and gives outputs by calling testcase_* functions. +## Cases are found and run as for testmarktest_rule_basic, except that stdin is empty, and output works differently: +## +## - `testcase_read_myhunk NAME` prints the body of hunk CASE/NAME. `testcase_read_anyhunk NAME` prints the body of hunk NAME. +## - `testcase_output NAME` reads its stdin, which testmarktest_run checks against hunk CASE/NAME, using testmark_check. +## NAME is always in the case: no leading `/`, and no whitespace. +## - Anything FN prints otherwise, and a nonzero exit, have no hunk to go in, so they fail the case. +## They're shown as a diff from nothing, with an `exit status: N` line; regen can't fix that. +## - `testcase_withtmpdir` makes a temp dir, and makes it FN's working directory; it's removed when FN ends. +## Take what you need from the old working directory first: `local here=$PWD`. +## +## Each of these prints a message and returns nonzero on misuse, or a hunk that isn't there. +## They exist only inside FN; fd 3 inside FN is theirs too, so nothing else there may write to it. +## +## - @param $1 GLOB: a bash pattern, matched against whole hunk names; `*` matches `/` too. +## - @param $2 FN: the name of a defined function. +testmarktest_rule() { _testmark_call _testmarktest_rule "$@"; } + ## Sends more functions into every case, for case functions that call them. ## ## - @param $@ names of defined functions. diff --git a/lib/testmark/run.bash b/lib/testmark/run.bash index fb42401..3d7718b 100644 --- a/lib/testmark/run.bash +++ b/lib/testmark/run.bash @@ -1,9 +1,10 @@ ## run.bash: the high road -- rules that make hunks into cases, and a runner that exercises each case and checks its output. ## Sourced by testmark.bash; the functions here assume they run inside `_testmark_call`. -# Rules, in the order made, as parallel arrays: rule i is glob i and function i. +# Rules, in the order made, as parallel arrays: rule i is glob i, function i, and road i, `basic` or `full`. _testmarktest_globs=() _testmarktest_fns=() +_testmarktest_roads=() # Functions sent into every case, besides the rule's own. _testmarktest_provided=() # A case's script sources function i from pinned fd (this base + i): the case's own function, then the provided ones, in order. @@ -12,6 +13,8 @@ _testmarktest_fdbase=10 # The directory of the script being run, absolute; empty if bash isn't running a script file. # Set when the library is sourced, so a later `cd` can't move it. _testmarktest_dir= +# The text of testcase.bash, which every case of testmarktest_rule gets. Status 1 is the end of the file. +IFS= read -r -d '' _testmarktest_testcase <"${BASH_SOURCE[0]%/*}/testcase.bash" || : ## Sets _testmarktest_dir to the directory of script $1, if it's a file. _testmarktest_init() { @@ -31,18 +34,24 @@ _testmarktest_need_fn() { } } -_testmarktest_rule_basic() { - (($# == 2)) || { - _testmark_err "usage: testmarktest_rule_basic GLOB FN" +_testmarktest_rule_basic() { _testmarktest_add_rule basic testmarktest_rule_basic "$@"; } + +_testmarktest_rule() { _testmarktest_add_rule full testmarktest_rule "$@"; } + +## Adds a rule for road $1; $2 is the public function asking, for messages; the rest are its arguments. +_testmarktest_add_rule() { + (($# == 4)) || { + _testmark_err "usage: $2 GLOB FN" return 1 } - [[ -n $1 ]] || { - _testmark_err "testmarktest_rule_basic: the glob is empty" + [[ -n $3 ]] || { + _testmark_err "$2: the glob is empty" return 1 } - _testmarktest_need_fn testmarktest_rule_basic "$2" || return 1 - _testmarktest_globs+=("$1") - _testmarktest_fns+=("$2") + _testmarktest_need_fn "$2" "$4" || return 1 + _testmarktest_globs+=("$3") + _testmarktest_fns+=("$4") + _testmarktest_roads+=("$1") } _testmarktest_provide() { @@ -67,7 +76,7 @@ _testmarktest_run() { return 1 } ((${#_testmarktest_fns[@]} > 0)) || { - _testmark_err "testmarktest_run: no rules; make them with testmarktest_rule_basic first" + _testmark_err "testmarktest_run: no rules; make them with testmarktest_rule_basic or testmarktest_rule first" return 1 } [[ -n $_testmarktest_dir ]] || { @@ -97,14 +106,15 @@ _testmarktest_doc() { return 1 } local -a names - local name i failed=0 any= + local name i failed=0 any='' index='' blob='' mapfile -t names < <(_testmark_names "$h") + _testmarktest_doc_data "$h" "${names[@]}" for name in "${names[@]}"; do for i in "${!_testmarktest_globs[@]}"; do # shellcheck disable=SC2053 # the glob is a pattern on purpose [[ $name == ${_testmarktest_globs[i]} ]] || continue matched[i]=$((matched[i] + 1)) any=1 - if _testmarktest_case "$h" "$name" "${_testmarktest_fns[i]}"; then + if "_testmarktest_case_${_testmarktest_roads[i]}" "$h" "$name" "${_testmarktest_fns[i]}"; then printf 'PASS %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}" else printf 'FAIL %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}" @@ -121,7 +131,7 @@ _testmarktest_doc() { } ## Exercises the case of trigger $2 in document $1 with function $3, and checks its output against the case's `output` hunk. -_testmarktest_case() { +_testmarktest_case_basic() { [[ $2 == */* ]] || { _testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case" return 1 @@ -158,10 +168,18 @@ _testmarktest_exec() { } ## Prints the script a case runs: strict mode, the functions it may call, and a call to function $1. +## For the full road, given the case's name as $2, also the testcase_* functions, the doc's hunks, and the frames back to the runner; +## see _testmarktest_case_full. ## This is the one place a case's script is built. _testmarktest_script() { local fn fd=$_testmarktest_fdbase printf 'set -euo pipefail\n' + if (($# == 2)); then + printf '%s\n' "$_testmarktest_testcase" + printf '_testcase_case=%q\n' "$2" + # Hunks arrive on fds 8 and 9 as raw bytes: $(<...) reads them in blocks and parses nothing, where script text is read a byte at a time. + printf '%s\n' '_testcase_index=$(</dev/fd/8)' '_testcase_blob=$(</dev/fd/9)' '_testcase_blob=${_testcase_blob%.}' 'exec 8<&- 9<&- 3>&1' '{' + fi # Each function is sourced from its own pinned fd, so bash's errors inside one name a source the # harness chose, whatever fds the environment holds open; _testmarktest_relabel maps it back. for fn in "$1" "${_testmarktest_provided[@]}"; do @@ -169,6 +187,99 @@ _testmarktest_script() { fd=$((fd + 1)) done printf '%q\n' "$1" + (($# == 1)) || printf '} 2>&1 | _testcase_stream\n' +} + +## Sets the caller's `index` and `blob` to what the full road's cases read hunks from: +## `blob` is every body of document $1 in a row, and `index` has a line "NAME OFFSET LENGTH" for each, in bytes. +## +## - @param $1 handle +## - @param $2... its hunk names +_testmarktest_doc_data() { + local h=$1 name body off=0 + local -a bodies=() + shift + index= + for name; do + _testmark_get_body "$h" "$name" + bodies+=("$body") + index+="$name $off ${#body}"$'\n' + off=$((off + ${#body})) + done + printf -v blob '%s' "${bodies[@]}" # one join, not an append per hunk, each of which would copy all before it +} + +## Exercises the case of trigger $2 in document $1 with function $3, the full road's way: +## each output the case gives with testcase_output is checked against its hunk, and anything else it prints, or a nonzero exit, fails it. +## Uses the caller's `index` and `blob`, from _testmarktest_doc_data. +## +## The case's stdout carries frames: `o LENGTH NAME` and a newline, then that many bytes, for each testcase_output; +## then `s LENGTH` and the rest of what it printed. The `x STATUS` line after them is the runner's own. +_testmarktest_case_full() { + [[ $2 == */* ]] || { + _testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case" + return 1 + } + local prefix=${2%/*} kind len name body text stream='' st='' i failed=0 garbled='' + local -a outs=() bodies=() + while IFS=' ' read -r kind len name; do + # Checked before any arithmetic, which would run code in a garbled length. + [[ $kind == [osx] && -n $len && $len != *[!0-9]* ]] || { + garbled=1 + break + } + body= + # On 4.4, `read -N 0` reads to the end. + if [[ $kind != x ]] && ((len > 0)); then + IFS= read -r -N "$len" body || { + garbled=1 + break + } + fi + case $kind in + o) outs+=("$name") bodies+=("$body") ;; + s) stream=$body ;; + x) st=$len ;; + esac + done < <( + _testmarktest_exec_full "$1" "$2" "$3" "$prefix" + printf 'x %s\n' "$?" + ) + [[ -z $garbled && -n $st ]] || { + _testmark_hunk_err "$1" "$2" "the case's report to the runner was garbled or cut short; nothing else in a case may write to fd 3" + return 1 + } + + if [[ -z ${TESTMARK_RAW_ERRORS+set} ]]; then + _testmarktest_relabel stream "$3" + for i in "${!bodies[@]}"; do + text=${bodies[i]} + _testmarktest_relabel text "$3" + bodies[i]=$text + done + fi + if ((st != 0)); then + [[ -z $stream || $stream == *$'\n' ]] || stream+=$'\n' + stream+="exit status: $st"$'\n' + fi + if [[ -n $stream ]]; then + local -n file=_testmark_file_$1 + _testmark_hunk_err "$1" "$2" "the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it" + >&2 diff -u --label "$file: $2 (expected nothing)" --label "$file: $2 (actual)" /dev/null <(printf '%s' "$stream") + failed=1 + fi + for i in "${!outs[@]}"; do + _testmark_check "$1" "$prefix/${outs[i]}" < <(printf '%s' "${bodies[i]}") || failed=1 + done + return "$failed" +} + +## Runs function $3 in a fresh bash, in the script's directory, as a case named $4 of the full road, with stdin from /dev/null. +## Its stdout is the frames; its stderr is the test's, for the case's bash's own trouble outside the function. +_testmarktest_exec_full() { + unset -v LC_ALL BASH_ENV # as in _testmarktest_exec + cd "$_testmarktest_dir" || return + "$BASH" <(_testmarktest_script "$3" "$4") 8< <(printf '%s' "$index") 9< <(printf '%s.' "$blob") </dev/null } ## Relabels bash's own runtime error messages in a case's output, held by the variable named $1: diff --git a/lib/testmark/testcase.bash b/lib/testmark/testcase.bash new file mode 100644 index 0000000..8d3288c --- /dev/null +++ b/lib/testmark/testcase.bash @@ -0,0 +1,77 @@ +## testcase.bash: the functions a case function of testmarktest_rule calls, to read hunks and give its outputs. +## Not sourced by testmark.bash: the runner sends this file's text into each case's own bash. +## They run under the case's `set -euo pipefail`, but may be called where errexit is ignored, so they check every status themselves. +## Their contract is documented with testmarktest_rule, in testmark.bash. +# shellcheck disable=SC2154 # _testcase_case, _testcase_index, and _testcase_blob are set by the case's script + +testcase_read_myhunk() { + (($# == 1)) || { + >&2 echo "usage: testcase_read_myhunk NAME" + return 1 + } + _testcase_read testcase_read_myhunk "$_testcase_case/$1" +} + +testcase_read_anyhunk() { + (($# == 1)) || { + >&2 echo "usage: testcase_read_anyhunk NAME" + return 1 + } + _testcase_read testcase_read_anyhunk "$1" +} + +## Prints the body of hunk $2; $1 is the public function asking, for the message. +_testcase_read() { + local - LC_ALL=C IFS=$'\n' entry at + set -f + # Each index line is "NAME OFFSET LENGTH", and names hold no whitespace, so "NAME " starts exactly one line. + for entry in $_testcase_index; do + [[ $entry == "$2 "* ]] || continue + at=${entry#"$2 "} + printf '%s' "${_testcase_blob:${at% *}:${at#* }}" + return + done + >&2 printf '%s: no hunk named "%s"\n' "$1" "$2" + return 1 +} + +testcase_output() { + local LC_ALL=C body + (($# == 1)) || { + >&2 echo "usage: testcase_output NAME <OUTPUT" + return 1 + } + [[ -n $1 && $1 != /* && $1 != *[[:space:]]* ]] || { + >&2 printf 'testcase_output: "%s" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace\n' "$1" + return 1 + } + # Status 0 means read stopped at a NUL, which no hunk can hold. + if IFS= read -r -d '' body; then + >&2 printf 'testcase_output: %s: output contains a NUL byte\n' "$1" + return 1 + fi + printf 'o %d %s\n%s' "${#body}" "$1" "$body" >&3 +} + +testcase_withtmpdir() { + local dir + (($# == 0)) || { + >&2 echo "usage: testcase_withtmpdir" + return 1 + } + dir=$(mktemp -d) || return 1 + # shellcheck disable=SC2064 # expanded now, on purpose: the trap outlives this function's locals + trap "rm -rf -- $(printf %q "$dir")" EXIT + cd "$dir" || return 1 +} + +## Reads the case function's stdout and stderr to the end, then sends them to the runner as one frame. +## The end comes only once everything that could write to them has exited, so this frame is the last. +_testcase_stream() { + local LC_ALL=C stream + if IFS= read -r -d '' stream; then + while IFS= read -r -d '' _; do :; done + stream+=$'\ntestmark: the output contains a NUL byte\n' + fi + printf 's %d\n%s' "${#stream}" "$stream" >&3 +} diff --git a/tests/test-run-full.sh b/tests/test-run-full.sh new file mode 100755 index 0000000..2a379a6 --- /dev/null +++ b/tests/test-run-full.sh @@ -0,0 +1,303 @@ +#!/bin/bash +## The full road: testmarktest_rule, and the testcase_* functions its case functions call. +## Each case writes a doc and a test script, then runs the script in a fresh bash, from another directory. +## These assert in plain bash, not through the runner itself, so a broken runner can't pass its own tests. +set -euo pipefail +cd "$(dirname "$0")" +# shellcheck source=tests/lib.sh +source ./lib.sh + +unset TESTMARK_REGEN TESTMARK_RAW_ERRORS + +## Prints a doc with one case, `a`: `a/in` holds "hello", `a/out` holds $1, and `a/side` holds $2; both empty or ending in a newline. +## A hunk outside the case, `shared/x`, holds "shared". +one() { + printf '[testmark]:# (shared/x)\n```\nshared\n```\n\n[testmark]:# (a/in)\n```\nhello\n```\n\n[testmark]:# (a/out)\n```\n%s```\n\n[testmark]:# (a/side)\n```\n%s```\n' "$1" "$2" +} + +# --- reading hunks, and outputs --- + +run_case "outputs are checked against the case's hunks" "$(one $'hello\n' $'shared\n')" ' +exercise_x() { + testcase_read_myhunk in | testcase_output out + testcase_read_anyhunk shared/x | testcase_output side +} +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +run_case "a mismatch fails, with a diff" "$(one $'bye\n' '')" ' +exercise_x() { testcase_read_myhunk in | testcase_output out; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md:11: hunk \"a/out\": output doesn't match +--- WORK/t/doc.md: a/out (expected) ++++ WORK/t/doc.md: a/out (actual) +@@ -1 +1 @@ +-bye ++hello +--- status 1" + +run_case "an output not given isn't checked" "$(one $'never\n' $'side\n')" ' +exercise_x() { echo side | testcase_output side; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +run_case "outputs are exact: empty, blank lines, no final newline, odd bytes" "$(one '' $'\n\n \\\\ $(x) `y` \xff\n')" ' +exercise_x() { + printf "" | testcase_output out + printf "\n\n \\\\\\\\ \$(x) \`y\` \xff" | testcase_output side +} +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +run_case "hunks read back exactly, whatever their names and bytes" '[testmark]:# (a/in) +``` +``` + +[testmark]:# (a/$(echo)`echo`["x"]'"'"'y) +``` + \\ $(x) `y` '$'\xff\r'' + +``` + +[testmark]:# (a/out) +``` + \\ $(x) `y` '$'\xff'' + +```' ' +exercise_x() { testcase_read_myhunk "\$(echo)\`echo\`[\"x\"]'"'"'y" | testcase_output out; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +# --- what the case may not print --- + +run_case "printing outside testcase_output fails the case; regen can't fix it" "$(one '' '')" ' +exercise_x() { echo stray; echo err >&2; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md +TESTMARK_REGEN=1 testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it +--- WORK/t/doc.md: a/in (expected nothing) ++++ WORK/t/doc.md: a/in (actual) +@@ -0,0 +1,2 @@ ++stray ++err +testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it +--- WORK/t/doc.md: a/in (expected nothing) ++++ WORK/t/doc.md: a/in (actual) +@@ -0,0 +1,2 @@ ++stray ++err +--- status 1" +expect_eq "and the doc is unchanged" "$(<t/doc.md)" "$(one '' '')" + +run_case "a nonzero exit fails the case, and outputs given before it are still checked" "$(one $'hello\n' '')" ' +exercise_x() { testcase_read_myhunk in | testcase_output out; false; echo unreached; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md:6: hunk \"a/in\": the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it +--- WORK/t/doc.md: a/in (expected nothing) ++++ WORK/t/doc.md: a/in (actual) +@@ -0,0 +1 @@ ++exit status: 1 +--- status 1" + +run_case "set -e works in a case, even with the run under ||" "$(one '' '')" ' +exercise_x() { false; echo unreached | testcase_output out; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md 2>/dev/null || echo failed' "FAIL doc.md: a/in (exercise_x) +failed +--- stderr + +--- status 0" + +run_case "a failing tool in a pipe into testcase_output fails the case" "$(one $'part\n' '')" ' +exercise_x() { { echo part; false; } | testcase_output out; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md 2>&1 | grep -F exit' "+exit status: 1 +--- stderr + +--- status 0" + +run_case "stdin is empty" "$(one $'[]\n' '')" ' +exercise_x() { echo "[$(cat)]" | testcase_output out; } +testmarktest_rule "*/in" exercise_x +echo leaked | testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +# --- the testcase_* functions refusing --- + +run_case "a missing hunk fails the case, naming it" "$(one '' '')" ' +exercise_x() { testcase_read_myhunk nope; } +exercise_y() { testcase_read_anyhunk in; } +testmarktest_rule "*/in" exercise_x +testmarktest_rule "*/in" exercise_y +testmarktest_run doc.md 2>&1 | grep -E "^(\+[^+]|FAIL)"' "+testcase_read_myhunk: no hunk named \"a/nope\" ++exit status: 1 +FAIL doc.md: a/in (exercise_x) ++testcase_read_anyhunk: no hunk named \"in\" ++exit status: 1 +FAIL doc.md: a/in (exercise_y) +--- stderr + +--- status 0" + +run_case "testcase_* functions refuse bad arguments" "$(one '' '')" ' +exercise_x() { + testcase_read_myhunk || : + testcase_read_anyhunk a b || : + testcase_output </dev/null || : + testcase_output /a/out </dev/null || : + testcase_output "a b" </dev/null || : + testcase_output "" </dev/null || : + printf "a\0b" | testcase_output out || : + testcase_withtmpdir x || : +} +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md 2>&1 | grep "^+[^+]"' '+usage: testcase_read_myhunk NAME ++usage: testcase_read_anyhunk NAME ++usage: testcase_output NAME <OUTPUT ++testcase_output: "/a/out" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace ++testcase_output: "a b" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace ++testcase_output: "" is not an output name: it names a hunk in the case, so it has no leading "/", and no whitespace ++testcase_output: out: output contains a NUL byte ++usage: testcase_withtmpdir +--- stderr + +--- status 0' + +run_case "a NUL byte printed outside testcase_output fails the case" "$(one '' '')" ' +exercise_x() { printf "a\0b\n"; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md 2>&1 | grep "^+[^+]"' '+a ++testmark: the output contains a NUL byte +--- stderr + +--- status 0' + +run_case "an output with no hunk fails the case" "$(one '' '')" ' +exercise_x() { echo x | testcase_output nope; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md: no hunk named \"a/nope\" +--- status 1" + +run_case "one output given twice, differently, fails the case" "$(one $'x\n' '')" ' +exercise_x() { echo x | testcase_output out; echo y | testcase_output out; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md:11: hunk \"a/out\": checked twice, with different output +--- status 1" + +run_case "a write to fd 3 garbles the report, and fails the case" "$(one '' '')" ' +exercise_x() { echo oops >&3; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_x) +--- stderr +testmark: WORK/t/doc.md:6: hunk \"a/in\": the case's report to the runner was garbled or cut short; nothing else in a case may write to fd 3 +--- status 1" + +# --- the working directory --- + +run_case "testcase_withtmpdir moves the case into a new dir, removed after" "$(one $'here: WORK/t\nin tmp: yes\nwrote: hi\n' '')" ' +exercise_x() { + local here=$PWD + testcase_withtmpdir + echo hi >f + { echo "here: $here"; [[ $PWD != "$here" ]] && echo "in tmp: yes"; echo "wrote: $(<f)"; } | sed "s|^here: .*/t$|here: WORK/t|" | testcase_output out + echo "$PWD" >"$here/tmpdir" +} +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md +[[ ! -e $(<"${BASH_SOURCE%/*}/tmpdir") ]] && echo removed' "PASS doc.md: a/in (exercise_x) +removed +--- stderr + +--- status 0" + +# --- regen, and mixing the roads --- + +run_case "regen writes every output given" "$(one '' '')" ' +exercise_x() { testcase_read_myhunk in | testcase_output out; echo s | testcase_output side; } +testmarktest_rule "*/in" exercise_x +TESTMARK_REGEN=1 testmarktest_run doc.md && testmarktest_run doc.md' "PASS doc.md: a/in (exercise_x) +PASS doc.md: a/in (exercise_x) +--- stderr + +--- status 0" +expect_eq "regen wrote the doc" "$(<t/doc.md)" "$(one $'hello\n' $'s\n')" + +run_case "both roads in one run" '[testmark]:# (a/in) +``` +hello +``` + +[testmark]:# (a/output) +``` +hello +``` + +[testmark]:# (b/script) +``` +``` + +[testmark]:# (b/out) +``` +hello +```' ' +exercise_cat() { cat; } +exercise_full() { testcase_read_anyhunk a/in | testcase_output out; } +testmarktest_rule_basic "*/in" exercise_cat +testmarktest_rule "*/script" exercise_full +testmarktest_run doc.md' "PASS doc.md: a/in (exercise_cat) +PASS doc.md: b/script (exercise_full) +--- stderr + +--- status 0" + +run_case "bash's errors are relabeled, in outputs and outside them" "$(one $'exercise_x: nope_a: command not found\n' '')" ' +exercise_x() { { nope_a || :; } 2>&1 | testcase_output out; nope_b; } +testmarktest_rule "*/in" exercise_x +testmarktest_run doc.md 2>&1 | grep -E "^(\+[^+]|FAIL)"' "+exercise_x: nope_b: command not found ++exit status: 127 +FAIL doc.md: a/in (exercise_x) +--- stderr + +--- status 0" + +run_case "the testcase_* functions don't exist outside a case" "$(one '' '')" ' +declare -F | grep -c testcase || :' "0 +--- stderr + +--- status 0" + +# --- arguments --- + +defined() { :; } +expect_fail "rule: one argument" 'testmark: usage: testmarktest_rule GLOB FN' testmarktest_rule '*/in' +expect_fail "rule: three arguments" 'testmark: usage: testmarktest_rule GLOB FN' testmarktest_rule '*/in' defined x +expect_fail "rule: an empty glob" 'testmark: testmarktest_rule: the glob is empty' testmarktest_rule '' defined +expect_fail "rule: not a function" 'testmark: testmarktest_rule: "cat" is not a defined function' testmarktest_rule '*/in' cat +expect_eq "refused calls leave no rules" "${#_testmarktest_fns[@]}" "0" + +exit "$RESULT" diff --git a/tests/test-run.sh b/tests/test-run.sh index c4728e7..8df6af3 100755 --- a/tests/test-run.sh +++ b/tests/test-run.sh @@ -377,7 +377,7 @@ testmark: WORK/t/nope.md: not a readable file run_case "a run with no rules is an error" "$(one '')" ' testmarktest_run doc.md' " --- stderr -testmark: testmarktest_run: no rules; make them with testmarktest_rule_basic first +testmark: testmarktest_run: no rules; make them with testmarktest_rule_basic or testmarktest_rule first --- status 1" # --- several rules ---