## run.bash: the high road -- rules that make hunks into cases, and a runner that exercises each case and checks its output. ## Sourced by testmark.bash; the functions here assume they run inside `_testmark_call`. # Rules, in the order made, as parallel arrays: rule i is glob i, function i, and road i, `basic` or `full`. _testmarktest_globs=() _testmarktest_fns=() _testmarktest_roads=() # Functions sent into every case, besides the rule's own. _testmarktest_provided=() # A case's script sources function i from pinned fd (this base + i): the case's own function, then the provided ones, in order. # _testmarktest_script assigns the pins; _testmarktest_relabel turns them back into function names. _testmarktest_fdbase=10 # The directory of the script being run, absolute; empty if bash isn't running a script file. # Set when the library is sourced, so a later `cd` can't move it. _testmarktest_dir= # testcase.bash's absolute path, for each case of testmarktest_rule to source. Absolute now, so neither a later `cd` nor the case's own directory can move it. _testmarktest_testcase=${BASH_SOURCE[0]%/*}/testcase.bash [[ $_testmarktest_testcase == /* ]] || _testmarktest_testcase=$PWD/$_testmarktest_testcase ## Sets _testmarktest_dir to the directory of script $1, if it's a file. _testmarktest_init() { local top=$1 [[ -f $top ]] || return 0 [[ $top == */* ]] || top=./$top _testmarktest_dir=${top%/*} _testmarktest_dir=${_testmarktest_dir:-/} [[ $_testmarktest_dir == /* ]] || _testmarktest_dir=$PWD/$_testmarktest_dir } ## Fails with a message unless $2 is a defined function; $1 is the public function asking, for the message. _testmarktest_need_fn() { declare -F -- "$2" >/dev/null || { _testmark_err "$1: \"$2\" is not a defined function" return 1 } } _testmarktest_rule_basic() { _testmarktest_add_rule basic testmarktest_rule_basic "$@"; } _testmarktest_rule() { _testmarktest_add_rule full testmarktest_rule "$@"; } ## Adds a rule for road $1; $2 is the public function asking, for messages; the rest are its arguments. _testmarktest_add_rule() { (($# == 4)) || { _testmark_err "usage: $2 GLOB FN" return 1 } [[ -n $3 ]] || { _testmark_err "$2: the glob is empty" return 1 } _testmarktest_need_fn "$2" "$4" || return 1 _testmarktest_globs+=("$3") _testmarktest_fns+=("$4") _testmarktest_roads+=("$1") } _testmarktest_provide() { (($# > 0)) || { _testmark_err "usage: testmarktest_provide FN..." return 1 } # The pins count up from _testmarktest_fdbase and must stay clear of the fds bash gives # process substitutions (63, counting down), so a case holds at most 50 functions. ((${#_testmarktest_provided[@]} + $# + 1 <= 50)) || { _testmark_err "testmarktest_provide: a case can hold at most 50 functions" return 1 } local fn for fn; do _testmarktest_need_fn testmarktest_provide "$fn" || return 1; done _testmarktest_provided+=("$@") } _testmarktest_run() { (($# > 0)) || { _testmark_err "usage: testmarktest_run DOC..." return 1 } ((${#_testmarktest_fns[@]} > 0)) || { _testmark_err "testmarktest_run: no rules; make them with testmarktest_rule_basic or testmarktest_rule first" return 1 } [[ -n $_testmarktest_dir ]] || { _testmark_err "testmarktest_run: no script file to find the docs beside; run it from a script" return 1 } local -a matched=() local i doc failed=0 for i in "${!_testmarktest_fns[@]}"; do matched[i]=0; done for doc; do _testmarktest_doc "$doc" || failed=1; done for i in "${!_testmarktest_fns[@]}"; do ((matched[i] > 0)) || { _testmark_err "rule \"${_testmarktest_globs[i]}\" ${_testmarktest_fns[i]}: matched no hunk" failed=1 } done return "$failed" } ## Runs every case in doc $1, reports each, and writes the doc's regen patches. ## Counts each rule's triggers in the caller's `matched`. _testmarktest_doc() { local doc=$1 path=$1 h=testmarktest_run [[ $path == /* ]] || path=$_testmarktest_dir/$path _testmark_parse "$h" "$path" || { printf 'FAIL %s\n' "$doc" return 1 } local -a names local name i failed=0 any='' index='' blob='' mapfile -t names < <(_testmark_names "$h") _testmarktest_doc_data "$h" "${names[@]}" for name in "${names[@]}"; do for i in "${!_testmarktest_globs[@]}"; do # shellcheck disable=SC2053 # the glob is a pattern on purpose [[ $name == ${_testmarktest_globs[i]} ]] || continue matched[i]=$((matched[i] + 1)) any=1 if "_testmarktest_case_${_testmarktest_roads[i]}" "$h" "$name" "${_testmarktest_fns[i]}"; then printf 'PASS %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}" else printf 'FAIL %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}" failed=1 fi done done [[ -n $any ]] || { _testmark_err "$doc: no hunk matches any rule" failed=1 } _testmark_write "$h" || failed=1 return "$failed" } ## Exercises the case of trigger $2 in document $1 with function $3, and checks its output against the case's `output` hunk. _testmarktest_case_basic() { [[ $2 == */* ]] || { _testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case" return 1 } local out=${2%/*}/output stream st _testmark_hunk "$1" "$out" || return 1 # The exit status follows the output after a NUL, which bash can't hold; so a second NUL means the output held one. { IFS= read -r -d '' stream if IFS= read -r -d '' st; then _testmark_hunk_err "$1" "$2" "output contains a NUL byte" return 1 fi } < <( _testmarktest_exec "$1" "$2" "$3" printf '\0%s' "$?" ) [[ -n ${TESTMARK_RAW_ERRORS+set} ]] || _testmarktest_relabel stream "$3" if ((st != 0)); then [[ -z $stream || $stream == *$'\n' ]] || stream+=$'\n' stream+="exit status: $st"$'\n' fi _testmark_check "$1" "$out" < <(printf '%s' "$stream") } ## Runs function $3 in a fresh bash, in the script's directory, with the body of hunk $2 of document $1 on stdin. ## stdout and stderr both go to stdout. Runs in a process substitution, so its `cd` and `unset` change nothing outside. _testmarktest_exec() { # _testmark_call's `local LC_ALL=C` would reach the case if the caller exported LC_ALL; unsetting it uncovers the caller's own. # BASH_ENV would be sourced by the case's bash, and a case should start clean. unset -v LC_ALL BASH_ENV cd "$_testmarktest_dir" 2>&1 || return "$BASH" <(_testmarktest_script "$3") < <(_testmark_body "$1" "$2") 2>&1 } ## Prints the script a case runs: strict mode, the functions it may call, and a call to function $1. ## For the full road, given the case's name as $2, also the testcase_* functions, the doc's hunks, and the frames back to the runner; ## see _testmarktest_case_full. ## This is the one place a case's script is built. _testmarktest_script() { local fn fd=$_testmarktest_fdbase printf 'set -euo pipefail\n' printf 'TESTCASE_DIR=%q\n' "$_testmarktest_dir" if (($# == 2)); then printf 'source %q\n' "$_testmarktest_testcase" printf '_testcase_case=%q\n' "$2" # Hunks arrive on fds 8 and 9 as raw bytes: $(<...) reads them in blocks and parses nothing, where script text is read a byte at a time. printf '%s\n' '_testcase_index=$(&1' '{' fi # Each function is sourced from its own pinned fd, so bash's errors inside one name a source the # harness chose, whatever fds the environment holds open; _testmarktest_relabel maps it back. for fn in "$1" "${_testmarktest_provided[@]}"; do printf 'source /dev/fd/%d %d< <(printf %%s %q)\n' "$fd" "$fd" "$(declare -f -- "$fn")" fd=$((fd + 1)) done printf '%q\n' "$1" (($# == 1)) || printf '} 2>&1 | _testcase_stream\n' } ## Sets the caller's `index` and `blob` to what the full road's cases read hunks from: ## `blob` is every body of document $1 in a row, and `index` has a line "NAME OFFSET LENGTH" for each, in bytes. ## ## - @param $1 handle ## - @param $2... its hunk names _testmarktest_doc_data() { local h=$1 name body off=0 local -a bodies=() shift index= for name; do _testmark_get_body "$h" "$name" bodies+=("$body") index+="$name $off ${#body}"$'\n' off=$((off + ${#body})) done printf -v blob '%s' "${bodies[@]}" # one join, not an append per hunk, each of which would copy all before it } ## Exercises the case of trigger $2 in document $1 with function $3, the full road's way: ## each output the case gives with testcase_output is checked against its hunk; ## anything else it prints, and a nonzero exit, go to its testcase_default_output if it named one, and otherwise fail it. ## Uses the caller's `index` and `blob`, from _testmarktest_doc_data. ## ## The case's stdout carries frames: `o LENGTH NAME` and a newline, then that many bytes, for each testcase_output; ## `d 0 NAME` for each testcase_default_output; then `s LENGTH` and the rest of what it printed. The `x STATUS` line after them is the runner's own. _testmarktest_case_full() { [[ $2 == */* ]] || { _testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case" return 1 } local prefix=${2%/*} kind len name body text stream='' st='' i failed=0 garbled='' default='' local -a outs=() bodies=() while IFS=' ' read -r kind len name; do # Checked before any arithmetic, which would run code in a garbled length. [[ $kind == [osxd] && -n $len && $len != *[!0-9]* ]] || { garbled=1 break } body= # On 4.4, `read -N 0` reads to the end. if [[ $kind == [os] ]] && ((len > 0)); then IFS= read -r -N "$len" body || { garbled=1 break } fi case $kind in o) outs+=("$name") bodies+=("$body") ;; d) default+=${default:+ }$name ;; s) stream=$body ;; x) st=$len ;; esac done < <( _testmarktest_exec_full "$1" "$2" "$3" "$prefix" printf 'x %s\n' "$?" ) [[ -z $garbled && -n $st ]] || { _testmark_hunk_err "$1" "$2" "the case's report to the runner was garbled or cut short; nothing else in a case may write to fd 3" return 1 } if [[ -z ${TESTMARK_RAW_ERRORS+set} ]]; then _testmarktest_relabel stream "$3" for i in "${!bodies[@]}"; do text=${bodies[i]} _testmarktest_relabel text "$3" bodies[i]=$text done fi if ((st != 0)); then [[ -z $stream || $stream == *$'\n' ]] || stream+=$'\n' stream+="exit status: $st"$'\n' fi if [[ $default == *' '* ]]; then _testmark_hunk_err "$1" "$2" "testcase_default_output was called more than once: $default" failed=1 elif [[ -n $default ]]; then _testmark_check "$1" "$prefix/$default" < <(printf '%s' "$stream") || failed=1 elif [[ -n $stream ]]; then local -n file=_testmark_file_$1 _testmark_hunk_err "$1" "$2" "the case printed this outside testcase_output, or failed, without a testcase_default_output for it to go to; regen can't take it" >&2 diff -u --label "$file: $2 (expected nothing)" --label "$file: $2 (actual)" /dev/null <(printf '%s' "$stream") failed=1 fi for i in "${!outs[@]}"; do _testmark_check "$1" "$prefix/${outs[i]}" < <(printf '%s' "${bodies[i]}") || failed=1 done return "$failed" } ## Runs function $3 in a fresh bash, in the script's directory, as a case named $4 of the full road, with stdin from /dev/null. ## Its stdout is the frames; its stderr is the test's, for the case's bash's own trouble outside the function. _testmarktest_exec_full() { unset -v LC_ALL BASH_ENV # as in _testmarktest_exec cd "$_testmarktest_dir" || return "$BASH" <(_testmarktest_script "$3" "$4") 8< <(printf '%s' "$index") 9< <(printf '%s.' "$blob")