run.bash

## run.bash: the high road -- rules that make hunks into cases, and a runner that exercises each case and checks its output.
## Sourced by testmark.bash; the functions here assume they run inside `_testmark_call`.

# Rules, in the order made, as parallel arrays: rule i is glob i, function i, and road i, `basic` or `full`.
_testmarktest_globs=()
_testmarktest_fns=()
_testmarktest_roads=()
# Functions sent into every case, besides the rule's own.
_testmarktest_provided=()
# A case's script sources function i from pinned fd (this base + i): the case's own function, then the provided ones, in order.
# _testmarktest_script assigns the pins; _testmarktest_relabel turns them back into function names.
_testmarktest_fdbase=10
# The directory of the script being run, absolute; empty if bash isn't running a script file.
# Set when the library is sourced, so a later `cd` can't move it.
_testmarktest_dir=
# testcase.bash's absolute path, for each case of testmarktest_rule to source.  Absolute now, so neither a later `cd` nor the case's own directory can move it.
_testmarktest_testcase=${BASH_SOURCE[0]%/*}/testcase.bash
[[ $_testmarktest_testcase == /* ]] || _testmarktest_testcase=$PWD/$_testmarktest_testcase

## Sets _testmarktest_dir to the directory of script $1, if it's a file.
_testmarktest_init() {
	local top=$1
	[[ -f $top ]] || return 0
	[[ $top == */* ]] || top=./$top
	_testmarktest_dir=${top%/*}
	_testmarktest_dir=${_testmarktest_dir:-/}
	[[ $_testmarktest_dir == /* ]] || _testmarktest_dir=$PWD/$_testmarktest_dir
}

## Fails with a message unless $2 is a defined function; $1 is the public function asking, for the message.
_testmarktest_need_fn() {
	declare -F -- "$2" >/dev/null || {
		_testmark_err "$1: \"$2\" is not a defined function"
		return 1
	}
}

_testmarktest_rule_basic() { _testmarktest_add_rule basic testmarktest_rule_basic "$@"; }

_testmarktest_rule() { _testmarktest_add_rule full testmarktest_rule "$@"; }

## Adds a rule for road $1; $2 is the public function asking, for messages; the rest are its arguments.
_testmarktest_add_rule() {
	(($# == 4)) || {
		_testmark_err "usage: $2 GLOB FN"
		return 1
	}
	[[ -n $3 ]] || {
		_testmark_err "$2: the glob is empty"
		return 1
	}
	_testmarktest_need_fn "$2" "$4" || return 1
	_testmarktest_globs+=("$3")
	_testmarktest_fns+=("$4")
	_testmarktest_roads+=("$1")
}

_testmarktest_provide() {
	(($# > 0)) || {
		_testmark_err "usage: testmarktest_provide FN..."
		return 1
	}
	# The pins count up from _testmarktest_fdbase and must stay clear of the fds bash gives
	# process substitutions (63, counting down), so a case holds at most 50 functions.
	((${#_testmarktest_provided[@]} + $# + 1 <= 50)) || {
		_testmark_err "testmarktest_provide: a case can hold at most 50 functions"
		return 1
	}
	local fn
	for fn; do _testmarktest_need_fn testmarktest_provide "$fn" || return 1; done
	_testmarktest_provided+=("$@")
}

_testmarktest_run() {
	(($# > 0)) || {
		_testmark_err "usage: testmarktest_run DOC..."
		return 1
	}
	((${#_testmarktest_fns[@]} > 0)) || {
		_testmark_err "testmarktest_run: no rules; make them with testmarktest_rule_basic or testmarktest_rule first"
		return 1
	}
	[[ -n $_testmarktest_dir ]] || {
		_testmark_err "testmarktest_run: no script file to find the docs beside; run it from a script"
		return 1
	}
	local -a matched=()
	local i doc failed=0
	for i in "${!_testmarktest_fns[@]}"; do matched[i]=0; done
	for doc; do _testmarktest_doc "$doc" || failed=1; done
	for i in "${!_testmarktest_fns[@]}"; do
		((matched[i] > 0)) || {
			_testmark_err "rule \"${_testmarktest_globs[i]}\" ${_testmarktest_fns[i]}: matched no hunk"
			failed=1
		}
	done
	return "$failed"
}

## Runs every case in doc $1, reports each, and writes the doc's regen patches.
## Counts each rule's triggers in the caller's `matched`.
_testmarktest_doc() {
	local doc=$1 path=$1 h=testmarktest_run
	[[ $path == /* ]] || path=$_testmarktest_dir/$path
	_testmark_parse "$h" "$path" || {
		printf 'FAIL  %s\n' "$doc"
		return 1
	}
	local -a names
	local name i failed=0 any='' index='' blob=''
	mapfile -t names < <(_testmark_names "$h")
	_testmarktest_doc_data "$h" "${names[@]}"
	for name in "${names[@]}"; do
		for i in "${!_testmarktest_globs[@]}"; do
			# shellcheck disable=SC2053 # the glob is a pattern on purpose
			[[ $name == ${_testmarktest_globs[i]} ]] || continue
			matched[i]=$((matched[i] + 1)) any=1
			if "_testmarktest_case_${_testmarktest_roads[i]}" "$h" "$name" "${_testmarktest_fns[i]}"; then
				printf 'PASS  %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}"
			else
				printf 'FAIL  %s: %s (%s)\n' "$doc" "$name" "${_testmarktest_fns[i]}"
				failed=1
			fi
		done
	done
	[[ -n $any ]] || {
		_testmark_err "$doc: no hunk matches any rule"
		failed=1
	}
	_testmark_write "$h" || failed=1
	return "$failed"
}

## Exercises the case of trigger $2 in document $1 with function $3, and checks its output against the case's `output` hunk.
_testmarktest_case_basic() {
	[[ $2 == */* ]] || {
		_testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case"
		return 1
	}
	local out=${2%/*}/output stream st
	_testmark_hunk "$1" "$out" || return 1
	# The exit status follows the output after a NUL, which bash can't hold; so a second NUL means the output held one.
	{
		IFS= read -r -d '' stream
		if IFS= read -r -d '' st; then
			_testmark_hunk_err "$1" "$2" "output contains a NUL byte"
			return 1
		fi
	} < <(
		_testmarktest_exec "$1" "$2" "$3"
		printf '\0%s' "$?"
	)
	[[ -n ${TESTMARK_RAW_ERRORS+set} ]] || _testmarktest_relabel stream "$3"
	if ((st != 0)); then
		[[ -z $stream || $stream == *$'\n' ]] || stream+=$'\n'
		stream+="exit status: $st"$'\n'
	fi
	_testmark_check "$1" "$out" < <(printf '%s' "$stream")
}

## Runs function $3 in a fresh bash, in the script's directory, with the body of hunk $2 of document $1 on stdin.
## stdout and stderr both go to stdout.  Runs in a process substitution, so its `cd` and `unset` change nothing outside.
_testmarktest_exec() {
	# _testmark_call's `local LC_ALL=C` would reach the case if the caller exported LC_ALL; unsetting it uncovers the caller's own.
	# BASH_ENV would be sourced by the case's bash, and a case should start clean.
	unset -v LC_ALL BASH_ENV
	cd "$_testmarktest_dir" 2>&1 || return
	"$BASH" <(_testmarktest_script "$3") < <(_testmark_body "$1" "$2") 2>&1
}

## Prints the script a case runs: strict mode, the functions it may call, and a call to function $1.
## For the full road, given the case's name as $2, also the testcase_* functions, the doc's hunks, and the frames back to the runner;
## see _testmarktest_case_full.
## This is the one place a case's script is built.
_testmarktest_script() {
	local fn fd=$_testmarktest_fdbase
	printf 'set -euo pipefail\n'
	printf 'TESTCASE_DIR=%q\n' "$_testmarktest_dir"
	if (($# == 2)); then
		printf 'source %q\n' "$_testmarktest_testcase"
		printf '_testcase_case=%q\n' "$2"
		# Hunks arrive on fds 8 and 9 as raw bytes: $(<...) reads them in blocks and parses nothing, where script text is read a byte at a time.
		printf '%s\n' '_testcase_index=$(</dev/fd/8)' '_testcase_blob=$(</dev/fd/9)' '_testcase_blob=${_testcase_blob%.}' 'exec 8<&- 9<&- 3>&1' '{'
	fi
	# Each function is sourced from its own pinned fd, so bash's errors inside one name a source the
	# harness chose, whatever fds the environment holds open; _testmarktest_relabel maps it back.
	for fn in "$1" "${_testmarktest_provided[@]}"; do
		printf 'source /dev/fd/%d %d< <(printf %%s %q)\n' "$fd" "$fd" "$(declare -f -- "$fn")"
		fd=$((fd + 1))
	done
	printf '%q\n' "$1"
	(($# == 1)) || printf '} 2>&1 | _testcase_stream\n'
}

## Sets the caller's `index` and `blob` to what the full road's cases read hunks from:
## `blob` is every body of document $1 in a row, and `index` has a line "NAME OFFSET LENGTH" for each, in bytes.
##
## - @param $1 handle
## - @param $2... its hunk names
_testmarktest_doc_data() {
	local h=$1 name body off=0
	local -a bodies=()
	shift
	index=
	for name; do
		_testmark_get_body "$h" "$name"
		bodies+=("$body")
		index+="$name $off ${#body}"$'\n'
		off=$((off + ${#body}))
	done
	printf -v blob '%s' "${bodies[@]}" # one join, not an append per hunk, each of which would copy all before it
}

## Exercises the case of trigger $2 in document $1 with function $3, the full road's way:
## each output the case gives with testcase_output is checked against its hunk, and anything else it prints, or a nonzero exit, fails it.
## Uses the caller's `index` and `blob`, from _testmarktest_doc_data.
##
## The case's stdout carries frames: `o LENGTH NAME` and a newline, then that many bytes, for each testcase_output;
## then `s LENGTH` and the rest of what it printed.  The `x STATUS` line after them is the runner's own.
_testmarktest_case_full() {
	[[ $2 == */* ]] || {
		_testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case"
		return 1
	}
	local prefix=${2%/*} kind len name body text stream='' st='' i failed=0 garbled=''
	local -a outs=() bodies=()
	while IFS=' ' read -r kind len name; do
		# Checked before any arithmetic, which would run code in a garbled length.
		[[ $kind == [osx] && -n $len && $len != *[!0-9]* ]] || {
			garbled=1
			break
		}
		body=
		# On 4.4, `read -N 0` reads to the end.
		if [[ $kind != x ]] && ((len > 0)); then
			IFS= read -r -N "$len" body || {
				garbled=1
				break
			}
		fi
		case $kind in
		o) outs+=("$name") bodies+=("$body") ;;
		s) stream=$body ;;
		x) st=$len ;;
		esac
	done < <(
		_testmarktest_exec_full "$1" "$2" "$3" "$prefix"
		printf 'x %s\n' "$?"
	)
	[[ -z $garbled && -n $st ]] || {
		_testmark_hunk_err "$1" "$2" "the case's report to the runner was garbled or cut short; nothing else in a case may write to fd 3"
		return 1
	}

	if [[ -z ${TESTMARK_RAW_ERRORS+set} ]]; then
		_testmarktest_relabel stream "$3"
		for i in "${!bodies[@]}"; do
			text=${bodies[i]}
			_testmarktest_relabel text "$3"
			bodies[i]=$text
		done
	fi
	if ((st != 0)); then
		[[ -z $stream || $stream == *$'\n' ]] || stream+=$'\n'
		stream+="exit status: $st"$'\n'
	fi
	if [[ -n $stream ]]; then
		local -n file=_testmark_file_$1
		_testmark_hunk_err "$1" "$2" "the case printed this outside testcase_output, or failed; there's no hunk for it, so regen can't take it"
		>&2 diff -u --label "$file: $2 (expected nothing)" --label "$file: $2 (actual)" /dev/null <(printf '%s' "$stream")
		failed=1
	fi
	for i in "${!outs[@]}"; do
		_testmark_check "$1" "$prefix/${outs[i]}" < <(printf '%s' "${bodies[i]}") || failed=1
	done
	return "$failed"
}

## Runs function $3 in a fresh bash, in the script's directory, as a case named $4 of the full road, with stdin from /dev/null.
## Its stdout is the frames; its stderr is the test's, for the case's bash's own trouble outside the function.
_testmarktest_exec_full() {
	unset -v LC_ALL BASH_ENV # as in _testmarktest_exec
	cd "$_testmarktest_dir" || return
	"$BASH" <(_testmarktest_script "$3" "$4") 8< <(printf '%s' "$index") 9< <(printf '%s.' "$blob") </dev/null
}

## Relabels bash's own runtime error messages in a case's output, held by the variable named $1:
## a line beginning `/dev/fd/F: line N: `, where F is the pin of case function $2 or of a provided function,
## trades that prefix for the function's name, becoming `FN: ` and the rest.
## The line number is dropped on purpose: it counts lines of `declare -f` output, text that exists in no file.
_testmarktest_relabel() {
	local -n _tmr_stream=$1
	# Most output holds none of bash's own errors, and is left as it is, without a walk.
	[[ $_tmr_stream == */dev/fd/* ]] || return 0
	local -a _tmr_fns=("$2" "${_testmarktest_provided[@]}") _tmr_lines
	local _tmr_i _tmr_j _tmr_prefix _tmr_body _tmr_num
	# Split once, keeping each line's newline: taking each line off the front of the string would copy the rest every time, quadratic in the output's size.
	mapfile _tmr_lines < <(printf '%s' "$_tmr_stream")
	for _tmr_i in "${!_tmr_lines[@]}"; do
		[[ ${_tmr_lines[_tmr_i]} == /dev/fd/* ]] || continue
		for _tmr_j in "${!_tmr_fns[@]}"; do
			_tmr_prefix="/dev/fd/$((_testmarktest_fdbase + _tmr_j)): line "
			[[ ${_tmr_lines[_tmr_i]} == "$_tmr_prefix"* ]] || continue
			_tmr_body=${_tmr_lines[_tmr_i]#"$_tmr_prefix"}
			_tmr_num=${_tmr_body%%:*}
			if [[ -n $_tmr_num && $_tmr_num != *[!0-9]* && $_tmr_body == "$_tmr_num: "* ]]; then
				_tmr_lines[_tmr_i]="${_tmr_fns[_tmr_j]}: ${_tmr_body#"$_tmr_num": }"
			fi
			break
		done
	done
	printf -v _tmr_stream '%s' "${_tmr_lines[@]}"
}