testmark-bash

Check layer: testmark_check compares output to a hunk, or under TESTMARK_REGEN stages it.

commit 4a4248cccf5b70c6fcce803cec7019ef983c724f · parent af11966ce4 · browse this tree
Ximianth, 2026-10-24 00:22:31 UTC

Follows testmark-rs's Checker (crates/testmark/src/check.rs):
- an absent hunk is an error in both modes; regen rewrites hunks, it doesn't add them.
- the first output checked against a hunk is recorded, matched or not,
  and a later check with different output is an error, in both modes.
- a mismatch returns 1, with a diff -u of expected and actual on stderr,
  labelled with the file and hunk name.
- in regen mode, a mismatch stages the output, keeping the opener line.

Unlike rs, output no hunk could hold is refused in both modes:
no final newline, or a CRLF (a body never has either, after parse).
rs regens such output on every run and never converges.

The record of what was checked is the "checked" kind, so testmark_write's
re-parse drops it with everything else.
The subshell guard applies in normal mode too, since the record is state.

The tests assert in plain bash, so a broken check layer can't vouch for itself,
and include a hunk of glob characters, since an unquoted right-hand side
in either comparison would make it a pattern.

diff --git a/lib/testmark.bash b/lib/testmark.bash
index e8e7c5e..050a507 100644
--- a/lib/testmark.bash
+++ b/lib/testmark.bash
@@ -1,4 +1,4 @@
-## testmark.bash: read and patch testmark documents (data hunks embedded in markdown) from bash.
+## testmark.bash: read and patch testmark documents (data hunks embedded in markdown) from bash, and check output against them.
 ##
 ## Source this file, by a path containing a slash; it sources the rest of the library from beside itself.
 ## Needs bash 4.4 or later; on older ones, sourcing fails.
@@ -16,6 +16,8 @@ fi
 source "${BASH_SOURCE[0]%/*}/testmark/document.bash"
 # shellcheck source=lib/testmark/patch.bash
 source "${BASH_SOURCE[0]%/*}/testmark/patch.bash"
+# shellcheck source=lib/testmark/check.bash
+source "${BASH_SOURCE[0]%/*}/testmark/check.bash"
 
 ## Runs an implementation function in a known environment, whatever the caller's settings.
 ##
@@ -70,3 +72,14 @@ testmark_patch() { _testmark_call _testmark_patch "$@"; }
 ##
 ## - @param $1 handle
 testmark_write() { _testmark_call _testmark_write "$@"; }
+
+## Checks output against a hunk's body, exactly.
+## With TESTMARK_REGEN set, to any value, a mismatch stages the output as the hunk's new body instead, for testmark_write.
+## In both modes, checking one hunk again with different output is an error.
+##
+## - @param $1 handle
+## - @param $2 hunk name
+## - @stdin the output: empty, or lines each ending in a newline, without CRLF.
+## - @return 0 if it matches, or if TESTMARK_REGEN staged it; 1 on a mismatch, or any error.
+## - @stderr on a mismatch, a line saying so, then a `diff -u` of the expected and actual output.
+testmark_check() { _testmark_call _testmark_check "$@"; }
diff --git a/lib/testmark/check.bash b/lib/testmark/check.bash
new file mode 100644
index 0000000..64f4c2d
--- /dev/null
+++ b/lib/testmark/check.bash
@@ -0,0 +1,42 @@
+## check.bash: checking output against a hunk, or under TESTMARK_REGEN, staging it as the hunk's new body.
+## Sourced by testmark.bash; the functions here assume they run inside `_testmark_call`.
+# shellcheck disable=SC2034,SC2178 # shellcheck does not follow namerefs; see document.bash
+
+_testmark_check() {
+	(($# == 2)) || {
+		_testmark_err "usage: testmark_check HANDLE NAME <ACTUAL"
+		return 1
+	}
+	_testmark_hunk "$1" "$2" || return 1
+	_testmark_same_shell "$1" || return 1
+	local input body
+	_testmark_input "$1: hunk \"$2\"" || return 1
+	# A body is "" or lines ending in LF, never CRLF, so other output could never match, and regen would rewrite it on every run.
+	[[ -z $input || ($input == *$'\n' && $input != *$'\r\n'*) ]] || {
+		_testmark_err "$1: hunk \"$2\": output must be empty or end in a newline, and contain no CRLF; normalize it before checking"
+		return 1
+	}
+
+	# Two checks claiming one hunk is a harness bug, whether or not the first one matched.
+	local -n checked=_testmark_checked_$1
+	if [[ -n ${checked[$2]+set} ]]; then
+		[[ ${checked[$2]} == "$input" ]] || {
+			_testmark_err "$1: hunk \"$2\": checked twice, with different output"
+			return 1
+		}
+		return 0
+	fi
+	checked[$2]=$input
+
+	_testmark_get_body "$1" "$2"
+	[[ $input != "$body" ]] || return 0
+	if [[ -n ${TESTMARK_REGEN+set} ]]; then
+		_testmark_stage "$1" "$2" "$input"
+		return
+	fi
+	local -n file=_testmark_file_$1
+	_testmark_err "$1: hunk \"$2\": output doesn't match; set TESTMARK_REGEN=1 to accept it"
+	# diff's status isn't checked: the comparison above has decided already, and diff is only here to show it.
+	>&2 diff -u --label "$file: $2 (expected)" --label "$file: $2 (actual)" <(printf '%s' "$body") <(printf '%s' "$input")
+	return 1
+}
diff --git a/tests/test-check.sh b/tests/test-check.sh
new file mode 100755
index 0000000..0f61e6f
--- /dev/null
+++ b/tests/test-check.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+## Checking output against hunks, and regenerating them under TESTMARK_REGEN.
+## These assert in plain bash, not through testmark_check itself, so a broken check layer can't pass its own tests.
+set -euo pipefail
+cd "$(dirname "$0")"
+# shellcheck source=tests/lib.sh
+source ./lib.sh
+
+DOC='[testmark]:# (a)\n```text\nalpha\n```\n\n[testmark]:# (empty)\n```\n```\n\n[testmark]:# (blank)\n```\n\n```\n\n[testmark]:# (glob)\n```\n*\n```\n'
+printf -- "$DOC" >c.md
+unset TESTMARK_REGEN
+
+## Runs a command in this shell, with TESTMARK_REGEN set.
+env_regen() { TESTMARK_REGEN=1 "$@"; }
+
+## Parses c.md afresh as handle c, checks one output against one hunk, and checks the status.
+##
+## - @param $1 description
+## - @param $2 hunk name
+## - @param $3 the output, as a printf format
+## - @param $4 the expected status
+check_case() {
+	local rc=0
+	testmark_parse c c.md
+	testmark_check c "$2" < <(printf -- "$3") 2>"$work/err" || rc=$?
+	expect_eq "$1" "$rc" "$4"
+}
+
+# --- normal mode: exact comparison ---
+check_case "match" a 'alpha\n' 0
+check_case "a wrong expectation fails" a 'omega\n' 1
+check_case "only the case differs" a 'ALPHA\n' 1
+check_case "one newline more" a 'alpha\n\n' 1
+check_case "empty hunk, empty output" empty '' 0
+check_case "empty hunk, one newline" empty '\n' 1
+check_case "one-blank-line hunk, one newline (diverges)" blank '\n' 0
+check_case "one-blank-line hunk, empty output" blank '' 1
+check_case "a body of glob characters matches only itself" glob 'x\n' 1
+check_case "and itself" glob '*\n' 0
+
+testmark_parse c c.md
+expect_fail "a mismatch shows a diff" "$(
+	cat <<-EOF
+		testmark: c: hunk "a": output doesn't match; set TESTMARK_REGEN=1 to accept it
+		--- $work/c.md: a (expected)
+		+++ $work/c.md: a (actual)
+		@@ -1 +1,2 @@
+		-alpha
+		+omega
+		+two
+	EOF
+)" testmark_check c a < <(printf 'omega\ntwo\n')
+expect_ok "and stages nothing" testmark_write c
+expect_eq "so the file is unchanged" "$(bytes c.md)" "$(esc "$(printf -- "$DOC")"$'\n')"
+
+# --- regen mode ---
+testmark_parse c c.md
+expect_ok "regen: a mismatch succeeds" env_regen testmark_check c a <<<'omega'
+expect_ok "regen: a match succeeds" env_regen testmark_check c blank <<<''
+expect_ok "regen: and the write" testmark_write c
+expect_eq "writes only the mismatch, keeping its info string" "$(bytes c.md)" \
+	"$(esc "$(printf -- "${DOC/alpha/omega}")"$'\n')"
+expect_ok "and then the same output checks clean" testmark_check c a <<<'omega'
+printf -- "$DOC" >c.md
+
+testmark_parse c c.md
+TESTMARK_REGEN='' expect_ok "regen: TESTMARK_REGEN set but empty is regen too" testmark_check c a <<<'omega'
+testmark_write c
+expect_eq "and writes" "$(testmark_body c a)" "omega"
+printf -- "$DOC" >c.md
+
+# --- both modes ---
+FENCE='body has a line starting with ```, which would end its code block'
+UNHOLDABLE='output must be empty or end in a newline, and contain no CRLF; normalize it before checking'
+for mode in normal regen; do
+	run=()
+	[[ $mode == normal ]] || run=(env_regen)
+	testmark_parse c c.md
+	expect_fail "$mode: absent hunk" 'testmark: c: no hunk named "nope"' "${run[@]}" testmark_check c nope <<<'x'
+	expect_fail "$mode: no final newline" "testmark: c: hunk \"a\": $UNHOLDABLE" "${run[@]}" testmark_check c a < <(printf 'omega')
+	expect_fail "$mode: CRLF" "testmark: c: hunk \"a\": $UNHOLDABLE" "${run[@]}" testmark_check c a < <(printf 'omega\r\n')
+	expect_fail "$mode: a NUL byte" 'testmark: c: hunk "a": input contains a NUL byte' "${run[@]}" testmark_check c a < <(printf 'a\0b\n')
+	expect_fail "$mode: arity" 'testmark: usage: testmark_check HANDLE NAME <ACTUAL' "${run[@]}" testmark_check c <<<'x'
+	expect_ok "$mode: none of which staged anything" testmark_write c
+	expect_eq "$mode: so the file is unchanged" "$(bytes c.md)" "$(esc "$(printf -- "$DOC")"$'\n')"
+
+	# Checking a hunk twice: the same output is fine, other output is an error, whether or not the first check matched.
+	for first in alpha omega; do
+		testmark_parse c c.md
+		"${run[@]}" testmark_check c a <<<"$first" 2>/dev/null || :
+		expect_ok "$mode: first $first, then the same again" "${run[@]}" testmark_check c a <<<"$first"
+		expect_fail "$mode: first $first, then other output" 'testmark: c: hunk "a": checked twice, with different output' "${run[@]}" testmark_check c a <<<'other'
+		expect_fail "$mode: first $first, then glob characters" 'testmark: c: hunk "a": checked twice, with different output' "${run[@]}" testmark_check c a <<<'*'
+	done
+done
+
+testmark_parse c c.md
+expect_fail "regen: output with a fence line" "testmark: c: hunk \"a\": $FENCE" env_regen testmark_check c a < <(printf 'x\n```\n')
+
+testmark_parse c c.md
+env_regen testmark_check c a <<<'omega'
+testmark_write c
+expect_fail "a write drops what was checked: other output is a mismatch, not a second check" "$(
+	cat <<-EOF
+		testmark: c: hunk "a": output doesn't match; set TESTMARK_REGEN=1 to accept it
+		--- $work/c.md: a (expected)
+		+++ $work/c.md: a (actual)
+		@@ -1 +1 @@
+		-omega
+		+other
+	EOF
+)" testmark_check c a <<<'other'
+printf -- "$DOC" >c.md
+
+# --- the subshell guard, in both modes ---
+SUBSHELL="testmark: c: called in a subshell (from a pipe, or inside \$(...)), where any change to the document is lost; instead of 'cmd | testmark_check HANDLE NAME', use 'testmark_check HANDLE NAME < <(cmd)'"
+check_from_pipe() { printf 'omega\n' | "$@" testmark_check c a; }
+check_in_subst() {
+	local got
+	# shellcheck disable=SC2034 # only the call inside $(...) matters, not its output
+	got=$("$@" testmark_check c a <<<'omega')
+}
+testmark_parse c c.md
+expect_fail "normal: from a pipe" "$SUBSHELL" check_from_pipe
+expect_fail "normal: inside \$(...)" "$SUBSHELL" check_in_subst
+expect_fail "regen: from a pipe" "$SUBSHELL" check_from_pipe env_regen
+expect_fail "regen: inside \$(...)" "$SUBSHELL" check_in_subst env_regen
+expect_ok "none of which staged anything" testmark_write c
+expect_eq "so the file is unchanged" "$(bytes c.md)" "$(esc "$(printf -- "$DOC")"$'\n')"
+expect_ok "regen with < <(...)" env_regen testmark_check c a < <(printf 'omega\n')
+
+# --- names with shell syntax: plain data through checking and regen ---
+names=('$(>pwned1)' '`>pwned2`' 'a]b' 'x[0]' '@' '*' '$HOME' "'q'" '"q"' 'a\b')
+for name in "${names[@]}"; do
+	printf '[testmark]:# (%s)\n```\nold\n```\n\n' "$name"
+done >syntax.md
+testmark_parse h syntax.md
+for name in "${names[@]}"; do env_regen testmark_check h "$name" <<<"new $name"; done
+for name in "${names[@]}"; do
+	expect_fail "checked $name twice" "testmark: h: hunk \"$name\": checked twice, with different output" testmark_check h "$name" <<<'other'
+done
+testmark_write h
+for name in "${names[@]}"; do
+	expect_ok "regenerated $name" testmark_check h "$name" <<<"new $name"
+done
+expect_eq "nothing was executed" "$(echo pwned*)" "pwned*"
+
+exit "$RESULT"
diff --git a/tests/test-patch.sh b/tests/test-patch.sh
index 15ef504..b0938d7 100755
--- a/tests/test-patch.sh
+++ b/tests/test-patch.sh
@@ -130,6 +130,7 @@ globals() {
 printf '[testmark]:# (x)\n```\nold\n```\n[testmark]:# (y)\n```\nold\n```\n' >g.md
 testmark_parse g g.md
 testmark_patch g x json <<<'new'
+testmark_check g y <<<'old'
 testmark_write g
 after_write=$(globals)
 testmark_parse g g.md