errors about a hunk name its file and annotation line, not the handle; and a mismatch no longer suggests regen.
commit 0c4d3ffbadaa521e614237660a38c6deffbca78c · parent 8ab33487b8
· browse this tree
Ximianth, 2026-10-24 20:04:22 UTC
A handle is a name the calling script picked; someone reading test output has never seen it. Hunk errors now read `testmark: FILE:LINE: hunk "NAME": ...`, in the FILE:LINE shape parse errors already use, with the hunk's annotation line, so editors and terminals can jump to it. An unknown hunk names the file. An unknown handle, and calling from a subshell, still name the handle: there's no file to name for the first, and the second is about the call, not the doc. The mismatch message dropped "set TESTMARK_REGEN=1 to accept it". Repeated per failure, it was noise; and regen rewrites every mismatch at once, so "accept it" undersold what it does, and cued accepting as the obvious response to a failure.
diff --git a/lib/testmark/check.bash b/lib/testmark/check.bash index 5d137b8..741f767 100644 --- a/lib/testmark/check.bash +++ b/lib/testmark/check.bash @@ -10,12 +10,12 @@ _testmark_check() { _testmark_hunk "$1" "$2" || return 1 _testmark_same_shell "$1" || return 1 local input body - _testmark_input "$1: hunk \"$2\"" || return 1 + _testmark_input "$1" "$2" || return 1 # Two checks claiming one hunk with different output is a harness bug, whether or not the first one matched. # The same output again is compared like the first time, so a mismatch can't pass by repetition. local -n checked=_testmark_checked_$1 [[ ${checked[$2]-$input} == "$input" ]] || { - _testmark_err "$1: hunk \"$2\": checked twice, with different output" + _testmark_hunk_err "$1" "$2" "checked twice, with different output" return 1 } checked[$2]=$input @@ -27,7 +27,7 @@ _testmark_check() { return fi local -n file=_testmark_file_$1 - _testmark_err "$1: hunk \"$2\": output doesn't match; set TESTMARK_REGEN=1 to accept it" + _testmark_hunk_err "$1" "$2" "output doesn't match" # diff's status isn't checked: the comparison above has decided already, and diff is only here to show it. >&2 diff -u --label "$file: $2 (expected)" --label "$file: $2 (actual)" <(printf '%s' "$body") <(printf '%s' "$input") return 1 diff --git a/lib/testmark/document.bash b/lib/testmark/document.bash index 396bda8..8bdb920 100644 --- a/lib/testmark/document.bash +++ b/lib/testmark/document.bash @@ -20,6 +20,13 @@ _testmark_err() { >&2 printf 'testmark: %s\n' "$1" } +## Prints an error about hunk $2 of document $1, which must both exist: "testmark: FILE:LINE: hunk "NAME": $3". +## LINE is the hunk's annotation line, so a reader can go straight to it. +_testmark_hunk_err() { + local -n file=_testmark_file_$1 start=_testmark_start_$1 + _testmark_err "$file:$((start[$2] - 1)): hunk \"$2\": $3" +} + ## Fails with a message unless $1 is a valid handle. _testmark_handle() { [[ -n $1 && $1 != *[!A-Za-z0-9_]* ]] || { @@ -135,7 +142,8 @@ _testmark_hunk() { _testmark_doc "$1" || return 1 local -n start=_testmark_start_$1 [[ -n ${start[$2]+set} ]] || { - _testmark_err "$1: no hunk named \"$2\"" + local -n file=_testmark_file_$1 + _testmark_err "$file: no hunk named \"$2\"" return 1 } } diff --git a/lib/testmark/patch.bash b/lib/testmark/patch.bash index ea80064..a400d85 100644 --- a/lib/testmark/patch.bash +++ b/lib/testmark/patch.bash @@ -16,11 +16,11 @@ _testmark_same_shell() { ## CRLF becomes LF, and a final newline is added if it's missing. ## So input reads as the body it would become, and a check compares like with like. ## -## - @param $1 what the input is for, as a prefix for the error message +## - @param $1 handle, and $2 hunk name, of the hunk the input is for, for the error message _testmark_input() { # Status 0 means read stopped at a NUL; see "NUL bytes vanish unless you check for them" in docs/hacking/bash-pitfalls.md. if IFS= read -r -d '' input; then - _testmark_err "$1: input contains a NUL byte" + _testmark_hunk_err "$1" "$2" "input contains a NUL byte" return 1 fi [[ $input != *$'\r\n'* ]] || input=${input//$'\r\n'/$'\n'} @@ -35,7 +35,7 @@ _testmark_stage() { local body=$3 opener=${4-} # Any line starting with ``` closes the code block, so such a body would end its hunk early and corrupt the rest of the doc. [[ $'\n'$body != *$'\n```'* ]] || { - _testmark_err "$1: hunk \"$2\": body has a line starting with \`\`\`, which would end its code block" + _testmark_hunk_err "$1" "$2" "body has a line starting with \`\`\`, which would end its code block" return 1 } local -n lines=_testmark_lines_$1 start=_testmark_start_$1 patch=_testmark_patch_$1 @@ -55,12 +55,12 @@ _testmark_patch() { if (($# == 3)); then # Parse trims info strings, and a newline would end the opener line, so neither would read back as given. [[ $3 != *$'\n'* && $3 != [[:space:]]* && $3 != *[[:space:]] ]] || { - _testmark_err "$1: hunk \"$2\": info string must be one line, without leading or trailing whitespace" + _testmark_hunk_err "$1" "$2" "info string must be one line, without leading or trailing whitespace" return 1 } opener='```'$3$'\n' fi - _testmark_input "$1: hunk \"$2\"" || return 1 + _testmark_input "$1" "$2" || return 1 _testmark_stage "$1" "$2" "$input" "$opener" } diff --git a/lib/testmark/run.bash b/lib/testmark/run.bash index b6f3904..ff22f6d 100644 --- a/lib/testmark/run.bash +++ b/lib/testmark/run.bash @@ -81,7 +81,6 @@ _testmarktest_run() { ## Runs every case in doc $1, reports each, and writes the doc's regen patches. ## Counts each rule's triggers in the caller's `matched`. _testmarktest_doc() { - # The handle shows in the check layer's messages, so it's named for who's speaking. local doc=$1 path=$1 h=testmarktest_run [[ $path == /* ]] || path=$_testmarktest_dir/$path _testmark_parse "$h" "$path" || { @@ -115,7 +114,7 @@ _testmarktest_doc() { ## Exercises the case of trigger $2 in document $1 with function $3, and checks its output against the case's `output` hunk. _testmarktest_case() { [[ $2 == */* ]] || { - _testmark_err "$1: hunk \"$2\": has no \"/\", so it names no case" + _testmark_hunk_err "$1" "$2" "has no \"/\", so it names no case" return 1 } local out=${2%/*}/output stream st @@ -124,7 +123,7 @@ _testmarktest_case() { { IFS= read -r -d '' stream if IFS= read -r -d '' st; then - _testmark_err "$1: hunk \"$2\": output contains a NUL byte" + _testmark_hunk_err "$1" "$2" "output contains a NUL byte" return 1 fi } < <( diff --git a/tests/test-api.sh b/tests/test-api.sh index 53e1389..d0ec45b 100755 --- a/tests/test-api.sh +++ b/tests/test-api.sh @@ -28,10 +28,10 @@ testmark_parse b b.md testmark_parse e empty.md expect_eq "handle a" "$(testmark_body a x)" "one" expect_eq "handle b" "$(testmark_body b x)" "two" -expect_fail "an empty doc doesn't see another doc's hunks" 'testmark: e: no hunk named "x"' testmark_body e x +expect_fail "an empty doc doesn't see another doc's hunks" "testmark: $work/empty.md: no hunk named \"x\"" testmark_body e x expect_ok "re-parse" testmark_parse a b.md expect_eq "re-parse replaces the doc" "$(testmark_body a x)" "two" -expect_fail "re-parse drops the old doc's names" 'testmark: a: no hunk named "old"' testmark_body a old +expect_fail "re-parse drops the old doc's names" "testmark: $work/b.md: no hunk named \"old\"" testmark_body a old expect_eq "re-parse leaves other handles alone" "$(testmark_body b x)" "two" expect_ok "re-parse of the same file" testmark_parse b b.md expect_eq "and it reads" "$(testmark_body b x)" "two" @@ -68,7 +68,7 @@ expect_eq "sourcing inside a function" "$( # --- errors --- testmark_parse r a.md expect_fail "unknown handle" 'testmark: unknown handle "nope"' testmark_body nope x -expect_fail "unknown hunk" 'testmark: r: no hunk named "nope"' testmark_info r nope +expect_fail "unknown hunk" "testmark: $work/a.md: no hunk named \"nope\"" testmark_info r nope expect_fail "invalid handle" 'testmark: invalid handle "a-b": must match [A-Za-z0-9_]+' testmark_parse a-b a.md expect_fail "empty handle" 'testmark: invalid handle "": must match [A-Za-z0-9_]+' testmark_parse '' a.md expect_fail "directory" 'testmark: .: not a readable file' testmark_parse r . @@ -88,7 +88,7 @@ expect_eq "names with shell syntax: list" "$(printf '%s\n' "${got[@]}")" "$(prin for name in "${names[@]}"; do expect_eq "name $name" "$(testmark_body h "$name")" "body of $name" done -expect_fail "an unknown name with shell syntax" 'testmark: h: no hunk named "$(>pwned3)"' testmark_body h '$(>pwned3)' +expect_fail "an unknown name with shell syntax" "testmark: $work/syntax.md: no hunk named \"\$(>pwned3)\"" testmark_body h '$(>pwned3)' expect_fail "a handle with shell syntax" 'testmark: invalid handle "h[$(>pwned4)]": must match [A-Za-z0-9_]+' testmark_body 'h[$(>pwned4)]' x expect_eq "nothing was executed" "$(echo pwned*)" "pwned*" diff --git a/tests/test-check.sh b/tests/test-check.sh index 7382760..f8dbcf2 100755 --- a/tests/test-check.sh +++ b/tests/test-check.sh @@ -48,7 +48,7 @@ check_case "empty output stays empty" empty '' 0 testmark_parse c c.md expect_fail "a mismatch shows a diff" "$( cat <<-EOF - testmark: c: hunk "a": output doesn't match; set TESTMARK_REGEN=1 to accept it + testmark: $work/c.md:1: hunk "a": output doesn't match --- $work/c.md: a (expected) +++ $work/c.md: a (actual) @@ -1 +1,2 @@ @@ -88,8 +88,8 @@ for mode in normal regen; do run=() [[ $mode == normal ]] || run=(env_regen) testmark_parse c c.md - expect_fail "$mode: absent hunk" 'testmark: c: no hunk named "nope"' "${run[@]}" testmark_check c nope <<<'x' - expect_fail "$mode: a NUL byte" 'testmark: c: hunk "a": input contains a NUL byte' "${run[@]}" testmark_check c a < <(printf 'a\0b\n') + expect_fail "$mode: absent hunk" "testmark: $work/c.md: no hunk named \"nope\"" "${run[@]}" testmark_check c nope <<<'x' + expect_fail "$mode: a NUL byte" "testmark: $work/c.md:1: hunk \"a\": input contains a NUL byte" "${run[@]}" testmark_check c a < <(printf 'a\0b\n') expect_fail "$mode: arity" 'testmark: usage: testmark_check HANDLE NAME <ACTUAL' "${run[@]}" testmark_check c <<<'x' expect_ok "$mode: none of which staged anything" testmark_write c expect_eq "$mode: so the file is unchanged" "$(bytes c.md)" "$(esc "$(printf -- "$DOC")"$'\n')" @@ -102,20 +102,20 @@ for mode in normal regen; do "${run[@]}" testmark_check c a <<<"$first" 2>/dev/null || rc=$? [[ $mode == regen || $first == alpha ]] || want=1 expect_eq "$mode: first $first, then the same again" "$rc" "$want" - expect_fail "$mode: first $first, then other output" 'testmark: c: hunk "a": checked twice, with different output' "${run[@]}" testmark_check c a <<<'other' - expect_fail "$mode: first $first, then glob characters" 'testmark: c: hunk "a": checked twice, with different output' "${run[@]}" testmark_check c a <<<'*' + expect_fail "$mode: first $first, then other output" "testmark: $work/c.md:1: hunk \"a\": checked twice, with different output" "${run[@]}" testmark_check c a <<<'other' + expect_fail "$mode: first $first, then glob characters" "testmark: $work/c.md:1: hunk \"a\": checked twice, with different output" "${run[@]}" testmark_check c a <<<'*' done done testmark_parse c c.md -expect_fail "regen: output with a fence line" "testmark: c: hunk \"a\": $FENCE" env_regen testmark_check c a < <(printf 'x\n```\n') +expect_fail "regen: output with a fence line" "testmark: $work/c.md:1: hunk \"a\": $FENCE" env_regen testmark_check c a < <(printf 'x\n```\n') testmark_parse c c.md env_regen testmark_check c a <<<'omega' testmark_write c expect_fail "a write drops what was checked: other output is a mismatch, not a second check" "$( cat <<-EOF - testmark: c: hunk "a": output doesn't match; set TESTMARK_REGEN=1 to accept it + testmark: $work/c.md:1: hunk "a": output doesn't match --- $work/c.md: a (expected) +++ $work/c.md: a (actual) @@ -1 +1 @@ @@ -149,8 +149,10 @@ for name in "${names[@]}"; do done >syntax.md testmark_parse h syntax.md for name in "${names[@]}"; do env_regen testmark_check h "$name" <<<"new $name"; done -for name in "${names[@]}"; do - expect_fail "checked $name twice" "testmark: h: hunk \"$name\": checked twice, with different output" testmark_check h "$name" <<<'other' +# Each name's annotation is on line 5i+1 of syntax.md. +for i in "${!names[@]}"; do + name=${names[i]} + expect_fail "checked $name twice" "testmark: $work/syntax.md:$((5 * i + 1)): hunk \"$name\": checked twice, with different output" testmark_check h "$name" <<<'other' done testmark_write h for name in "${names[@]}"; do diff --git a/tests/test-patch.sh b/tests/test-patch.sh index 717a575..e5a512c 100755 --- a/tests/test-patch.sh +++ b/tests/test-patch.sh @@ -87,7 +87,7 @@ expect_eq "staging again replaces all of what was staged, info string included" cp orig.md two.md testmark_parse t two.md testmark_patch t a <<<'staged' -expect_fail "a refused patch" 'testmark: t: hunk "a": body has a line starting with ```, which would end its code block' testmark_patch t a <<<'```' +expect_fail "a refused patch" "testmark: $work/two.md:1: hunk \"a\": body has a line starting with \`\`\`, which would end its code block" testmark_patch t a <<<'```' testmark_write t mapfile -t got < <(two_md '[testmark]:# (a)\n```\nstaged\n```\n\n[testmark]:# (b)\n```text\ntwo\n```\n') expect_eq "leaves what was staged before" "${got[0]}" "${got[1]}" @@ -142,18 +142,18 @@ printf '[testmark]:# (x)\n```\nold\n```\n' >r.md testmark_parse r r.md FENCE='body has a line starting with ```, which would end its code block' INFO='info string must be one line, without leading or trailing whitespace' -expect_fail "unknown hunk" 'testmark: r: no hunk named "nope"' testmark_patch r nope <<<'x' +expect_fail "unknown hunk" "testmark: $work/r.md: no hunk named \"nope\"" testmark_patch r nope <<<'x' expect_fail "unknown handle" 'testmark: unknown handle "nope"' testmark_patch nope x <<<'x' expect_fail "write: unknown handle" 'testmark: unknown handle "nope"' testmark_write nope -expect_fail "a fence line first" "testmark: r: hunk \"x\": $FENCE" testmark_patch r x < <(printf '```\nx\n') -expect_fail "a fence line later" "testmark: r: hunk \"x\": $FENCE" testmark_patch r x < <(printf 'x\n```json\n') -expect_fail "a fence line without its newline" "testmark: r: hunk \"x\": $FENCE" testmark_patch r x < <(printf 'x\n```') -expect_fail "a four-backtick fence line" "testmark: r: hunk \"x\": $FENCE" testmark_patch r x < <(printf '````\n') -expect_fail "info: a newline" "testmark: r: hunk \"x\": $INFO" testmark_patch r x $'a\nb' <<<'x' -expect_fail "info: a final newline" "testmark: r: hunk \"x\": $INFO" testmark_patch r x $'a\n' <<<'x' -expect_fail "info: leading space" "testmark: r: hunk \"x\": $INFO" testmark_patch r x ' a' <<<'x' -expect_fail "info: trailing tab" "testmark: r: hunk \"x\": $INFO" testmark_patch r x $'a\t' <<<'x' -expect_fail "a NUL byte" 'testmark: r: hunk "x": input contains a NUL byte' testmark_patch r x < <(printf 'a\0b\n') +expect_fail "a fence line first" "testmark: $work/r.md:1: hunk \"x\": $FENCE" testmark_patch r x < <(printf '```\nx\n') +expect_fail "a fence line later" "testmark: $work/r.md:1: hunk \"x\": $FENCE" testmark_patch r x < <(printf 'x\n```json\n') +expect_fail "a fence line without its newline" "testmark: $work/r.md:1: hunk \"x\": $FENCE" testmark_patch r x < <(printf 'x\n```') +expect_fail "a four-backtick fence line" "testmark: $work/r.md:1: hunk \"x\": $FENCE" testmark_patch r x < <(printf '````\n') +expect_fail "info: a newline" "testmark: $work/r.md:1: hunk \"x\": $INFO" testmark_patch r x $'a\nb' <<<'x' +expect_fail "info: a final newline" "testmark: $work/r.md:1: hunk \"x\": $INFO" testmark_patch r x $'a\n' <<<'x' +expect_fail "info: leading space" "testmark: $work/r.md:1: hunk \"x\": $INFO" testmark_patch r x ' a' <<<'x' +expect_fail "info: trailing tab" "testmark: $work/r.md:1: hunk \"x\": $INFO" testmark_patch r x $'a\t' <<<'x' +expect_fail "a NUL byte" "testmark: $work/r.md:1: hunk \"x\": input contains a NUL byte" testmark_patch r x < <(printf 'a\0b\n') expect_fail "patch arity, too few" 'testmark: usage: testmark_patch HANDLE NAME [INFO] <BODY' testmark_patch r <<<'x' expect_fail "patch arity, too many" 'testmark: usage: testmark_patch HANDLE NAME [INFO] <BODY' testmark_patch r x json more <<<'x' expect_fail "write arity" 'testmark: usage: testmark_write HANDLE' testmark_write r r diff --git a/tests/test-run.sh b/tests/test-run.sh index 60bd848..3cfd6ab 100755 --- a/tests/test-run.sh +++ b/tests/test-run.sh @@ -55,7 +55,7 @@ exercise_cat() { cat; } testmarktest_rule_basic "*/in" exercise_cat testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_cat) --- stderr -testmark: testmarktest_run: hunk \"a/output\": output doesn't match; set TESTMARK_REGEN=1 to accept it +testmark: WORK/t/doc.md:6: hunk \"a/output\": output doesn't match --- WORK/t/doc.md: a/output (expected) +++ WORK/t/doc.md: a/output (actual) @@ -1 +1 @@ @@ -116,7 +116,7 @@ exercise_nul() { printf "a\0b\n"; } testmarktest_rule_basic "*/in" exercise_nul testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_nul) --- stderr -testmark: testmarktest_run: hunk \"a/in\": output contains a NUL byte +testmark: WORK/t/doc.md:1: hunk \"a/in\": output contains a NUL byte --- status 1" # --- the case's process --- @@ -180,7 +180,7 @@ testmarktest_run doc.md echo unreached' "FAIL doc.md: a/in (exercise_cat) PASS doc.md: b/in (exercise_cat) --- stderr -testmark: testmarktest_run: hunk \"a/output\": output doesn't match; set TESTMARK_REGEN=1 to accept it +testmark: WORK/t/doc.md:5: hunk \"a/output\": output doesn't match --- WORK/t/doc.md: a/output (expected) +++ WORK/t/doc.md: a/output (actual) @@ -1 +0,0 @@ @@ -313,8 +313,8 @@ testmarktest_run doc.md TESTMARK_REGEN=1 testmarktest_run doc.md' "FAIL doc.md: a/in (exercise_cat) FAIL doc.md: a/in (exercise_cat) --- stderr -testmark: testmarktest_run: no hunk named \"a/output\" -testmark: testmarktest_run: no hunk named \"a/output\" +testmark: WORK/t/doc.md: no hunk named \"a/output\" +testmark: WORK/t/doc.md: no hunk named \"a/output\" --- status 1" run_case "a trigger without a / fails" '[testmark]:# (in) @@ -324,7 +324,7 @@ exercise_cat() { cat; } testmarktest_rule_basic "*in" exercise_cat testmarktest_run doc.md' "FAIL doc.md: in (exercise_cat) --- stderr -testmark: testmarktest_run: hunk \"in\": has no \"/\", so it names no case +testmark: WORK/t/doc.md:1: hunk \"in\": has no \"/\", so it names no case --- status 1" run_case "a rule matching nothing fails the run, after every case runs" "$(one $'hello\n')" '