testmark-rs

testmark: add the checking layer (Checker, read_file, write_back)

commit 20c0b098eacac69f739562a7f8b52de262e16e6b · parent 9049dd4f1c · browse this tree
Ximianth, 2026-08-10 18:31:21 UTC

The testing layer, sitting on Document and PatchAccumulator: check()
compares an actual value against a hunk body, and in regen mode
(TESTMARK_REGEN in the env, captured at construction) records a patch
instead of failing. Downstream harnesses get one call with no env-var
branching, and the expected value drops out of their signatures entirely:
the document knows it, so a caller can no longer pass a copy that
predates an earlier patch.

Deliberate semantics:

- An absent hunk is an error in both modes: regen rewrites hunks,
  it does not invent them.
- Checking one hunk twice with different values is a Conflict error
  in both modes: two tests claiming one hunk is a harness bug, and
  last-write-wins would hide it. Claims are tracked separately from
  patches, so this holds even when the first check matched the document.
- No thread-awareness: a parallel harness wraps the Checker in a Mutex
  at the point where testmark meets the parallel runner, and
  single-threaded users pay nothing.
- No file locking: one Checker per document per process is the contract;
  write_back is the caller's explicit final step (and a no-op when no
  patches accumulated, so harnesses call it unconditionally).

read_file/write_back are free functions: Document stays ignorant of
where its text lives.

diff --git a/crates/testmark/src/check.rs b/crates/testmark/src/check.rs
new file mode 100644
index 0000000..812bb35
--- /dev/null
+++ b/crates/testmark/src/check.rs
@@ -0,0 +1,201 @@
+//! Assert-or-regen over one document: the testing layer.
+//!
+//! `Checker` wraps a [`Document`] and a [`PatchAccumulator`] (both pub fields;
+//! this layer adds behavior, it doesn't hide anything).
+//! In normal mode, [`Checker::check`] compares an actual value against a hunk's body.
+//! In regen mode -- captured at construction, from the `TESTMARK_REGEN` env var --
+//! a difference is recorded as a pending patch instead of failing.
+//! Writing the patched text back out is the caller's job: see [`write_back`].
+//!
+//! # Use with test harnesses
+//!
+//! One `Checker` per document per process; the crate does no file locking,
+//! so two *processes* regenerating the same file concurrently will lose writes.
+//! (For `cargo test`, each integration-test binary is its own process:
+//! test binaries that share fixture files must not run regen concurrently.)
+//! Within one process, parallel tests can share a `Checker` by wrapping it
+//! in a `Mutex` at the point where testmark meets the parallel runner;
+//! the crate itself stays thread-unaware so single-threaded use pays nothing.
+//!
+//! Call `write_back` before anything that calls `process::exit`
+//! (for example, libtest_mimic's `Conclusion::exit`), or accumulated regen
+//! patches are silently dropped.
+
+use std::fmt;
+use std::path::{Path, PathBuf};
+
+use crate::document::Document;
+use crate::patch::{PatchAccumulator, PatchError};
+
+/// Assert-or-regen over one document.  See the module docs for the lifecycle.
+pub struct Checker {
+	pub doc: Document,
+	pub patches: PatchAccumulator,
+	regen: bool,
+	/// The first value each hunk was checked with, for conflict detection.
+	/// Kept separately from `patches`, which only holds actual differences:
+	/// a first check that *matched* the document still claims the hunk.
+	claims: Vec<(String, String)>,
+}
+
+impl Checker {
+	/// Regen mode iff the `TESTMARK_REGEN` env var is set (to any value).
+	pub fn from_env(doc: Document) -> Checker {
+		Checker::new(doc, std::env::var_os("TESTMARK_REGEN").is_some())
+	}
+
+	/// Explicit mode selection (mostly for testing harness code itself).
+	pub fn new(doc: Document, regen: bool) -> Checker {
+		Checker {
+			doc,
+			patches: PatchAccumulator::new(),
+			regen,
+			claims: Vec::new(),
+		}
+	}
+
+	/// Compare `actual` against the named code hunk's body.
+	///
+	/// Normal mode: a difference is [`CheckError::Mismatch`].
+	/// Regen mode: a difference records a patch and returns Ok.
+	/// Both modes: an absent hunk is an error -- regen rewrites hunks,
+	/// it does not invent them -- and a second check of the same hunk with a
+	/// *different* value is [`CheckError::Conflict`]: two tests claiming one
+	/// hunk is a harness bug, and last-write-wins here would hide it.
+	pub fn check(&mut self, hunk: &str, actual: &str) -> Result<(), CheckError> {
+		let h = self.doc.hunk(hunk).ok_or_else(|| CheckError::NoSuchHunk {
+			name: hunk.to_string(),
+		})?;
+		let expected = h.body().ok_or_else(|| CheckError::NotACodeHunk {
+			name: hunk.to_string(),
+		})?;
+		if let Some((_, earlier)) = self.claims.iter().find(|(name, _)| name == hunk) {
+			if earlier == actual {
+				return Ok(());
+			}
+			return Err(CheckError::Conflict {
+				name: hunk.to_string(),
+				earlier: earlier.clone(),
+				later: actual.to_string(),
+			});
+		}
+		self.claims.push((hunk.to_string(), actual.to_string()));
+		if actual == expected {
+			return Ok(());
+		}
+		if self.regen {
+			self.patches.put(hunk, actual);
+			return Ok(());
+		}
+		Err(CheckError::Mismatch {
+			name: hunk.to_string(),
+			expected: expected.to_string(),
+			actual: actual.to_string(),
+		})
+	}
+}
+
+/// Write the checker's patched document text to `path`, if any patches
+/// accumulated.  A no-op when there are none, so it's harmless (and cheap)
+/// outside regen mode: call it unconditionally at the end of a harness.
+pub fn write_back(path: &Path, chk: &Checker) -> Result<(), WriteBackError> {
+	if chk.patches.is_empty() {
+		return Ok(());
+	}
+	let text = chk.doc.patched_all(&chk.patches)?;
+	std::fs::write(path, text).map_err(|source| WriteBackError::Io {
+		path: path.to_path_buf(),
+		source,
+	})
+}
+
+/// Problems from [`Checker::check`].
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum CheckError {
+	/// The named hunk is not in the document (in regen mode too:
+	/// regen rewrites hunks, it does not invent them).
+	NoSuchHunk {
+		name: String,
+	},
+	/// The named hunk is a table, not a code hunk.
+	NotACodeHunk {
+		name: String,
+	},
+	/// Normal mode: the actual value differs from the hunk body.
+	Mismatch {
+		name: String,
+		expected: String,
+		actual: String,
+	},
+	/// The same hunk was checked twice with different values
+	/// (e.g. two tests claiming one hunk, or one test whose subject is
+	/// expected to be deterministic producing two different outputs).
+	Conflict {
+		name: String,
+		earlier: String,
+		later: String,
+	},
+}
+
+impl fmt::Display for CheckError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			CheckError::NoSuchHunk {
+				name,
+			} => write!(f, "no hunk named {name:?} in the document"),
+			CheckError::NotACodeHunk {
+				name,
+			} => write!(f, "hunk {name:?} is not a code hunk"),
+			CheckError::Mismatch {
+				name,
+				expected,
+				actual,
+			} => write!(
+				f,
+				"mismatch in {name}:\n--- expected ---\n{expected}--- actual ---\n{actual}--- end ---\n(run with TESTMARK_REGEN=1 to accept the actual output)"
+			),
+			CheckError::Conflict {
+				name,
+				earlier,
+				later,
+			} => write!(
+				f,
+				"hunk {name} checked twice with different values (a harness bug):\n--- earlier ---\n{earlier}--- later ---\n{later}--- end ---"
+			),
+		}
+	}
+}
+
+impl std::error::Error for CheckError {}
+
+/// Problems from [`write_back`].
+#[derive(Debug)]
+pub enum WriteBackError {
+	Patch(PatchError),
+	Io {
+		path: PathBuf,
+		source: std::io::Error,
+	},
+}
+
+impl From<PatchError> for WriteBackError {
+	fn from(e: PatchError) -> WriteBackError {
+		WriteBackError::Patch(e)
+	}
+}
+
+impl fmt::Display for WriteBackError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			WriteBackError::Patch(e) => write!(f, "{e}"),
+			WriteBackError::Io {
+				path,
+				source,
+			} => write!(f, "writing {}: {source}", path.display()),
+		}
+	}
+}
+
+impl std::error::Error for WriteBackError {}
+
+include!("check_test.rs");
diff --git a/crates/testmark/src/check_test.rs b/crates/testmark/src/check_test.rs
new file mode 100644
index 0000000..f8f54f5
--- /dev/null
+++ b/crates/testmark/src/check_test.rs
@@ -0,0 +1,138 @@
+#[cfg(test)]
+mod check_tests {
+	use super::*;
+
+	const DOC: &str = "\
+[testmark]:# (a/expect)\n\
+```\n\
+alpha\n\
+```\n\
+\n\
+[testmark]:# (b/expect)\n\
+```\n\
+beta\n\
+```\n\
+\n\
+[testmark]:# (tbl)\n\
+| h |\n\
+| --- |\n\
+| 1 |\n";
+
+	fn checker(regen: bool) -> Checker {
+		Checker::new(Document::parse(DOC).expect("parses"), regen)
+	}
+
+	#[test]
+	fn normal_mode_match_is_ok() {
+		let mut chk = checker(false);
+		assert_eq!(chk.check("a/expect", "alpha\n"), Ok(()));
+		assert!(chk.patches.is_empty());
+	}
+
+	#[test]
+	fn normal_mode_mismatch_is_error_with_regen_hint() {
+		let mut chk = checker(false);
+		let err = chk.check("a/expect", "omega\n").unwrap_err();
+		assert!(matches!(err, CheckError::Mismatch { .. }));
+		let msg = err.to_string();
+		assert!(msg.contains("mismatch in a/expect"));
+		assert!(msg.contains("alpha"));
+		assert!(msg.contains("omega"));
+		assert!(msg.contains("TESTMARK_REGEN=1"));
+		assert!(chk.patches.is_empty());
+	}
+
+	#[test]
+	fn regen_mode_mismatch_records_a_patch() {
+		let mut chk = checker(true);
+		assert_eq!(chk.check("a/expect", "omega\n"), Ok(()));
+		assert_eq!(chk.patches.get("a/expect"), Some("omega\n"));
+		let text = chk.doc.patched_all(&chk.patches).unwrap();
+		let redoc = Document::parse(&text).expect("reparses");
+		assert_eq!(redoc.hunk("a/expect").unwrap().body().unwrap(), "omega\n");
+		assert_eq!(redoc.hunk("b/expect").unwrap().body().unwrap(), "beta\n");
+	}
+
+	#[test]
+	fn regen_mode_match_records_nothing() {
+		let mut chk = checker(true);
+		assert_eq!(chk.check("a/expect", "alpha\n"), Ok(()));
+		assert!(chk.patches.is_empty());
+	}
+
+	#[test]
+	fn absent_hunk_is_error_in_both_modes() {
+		for regen in [false, true] {
+			let mut chk = checker(regen);
+			assert_eq!(
+				chk.check("no/such", "x\n"),
+				Err(CheckError::NoSuchHunk {
+					name: "no/such".into(),
+				})
+			);
+		}
+	}
+
+	#[test]
+	fn table_hunk_is_error() {
+		let mut chk = checker(false);
+		assert_eq!(
+			chk.check("tbl", "x\n"),
+			Err(CheckError::NotACodeHunk {
+				name: "tbl".into(),
+			})
+		);
+	}
+
+	#[test]
+	fn recheck_with_same_value_is_ok() {
+		for regen in [false, true] {
+			let mut chk = checker(regen);
+			assert_eq!(chk.check("a/expect", "alpha\n"), Ok(()));
+			assert_eq!(chk.check("a/expect", "alpha\n"), Ok(()));
+		}
+	}
+
+	#[test]
+	fn recheck_with_different_value_is_conflict() {
+		// Conflicts must surface in both modes, and regardless of whether the
+		// first check matched the document (i.e. whether a patch was recorded).
+		for regen in [false, true] {
+			for first in ["alpha\n", "omega\n"] {
+				let mut chk = checker(regen);
+				let _ = chk.check("a/expect", first);
+				let err = chk.check("a/expect", "something else\n").unwrap_err();
+				assert!(
+					matches!(err, CheckError::Conflict { .. }),
+					"regen={regen} first={first:?}: got {err:?}"
+				);
+			}
+		}
+	}
+
+	#[test]
+	fn write_back_roundtrip() {
+		let dir = std::env::temp_dir().join(format!("testmark-check-test-{}", std::process::id()));
+		std::fs::create_dir_all(&dir).expect("tmp dir");
+		let path = dir.join("fixture.md");
+		std::fs::write(&path, DOC).expect("seed fixture");
+
+		let mut chk = Checker::new(crate::read_file(&path).expect("reads"), true);
+		chk.check("b/expect", "gamma\n").expect("regen records");
+		write_back(&path, &chk).expect("writes");
+
+		let redoc = crate::read_file(&path).expect("re-reads");
+		assert_eq!(redoc.hunk("b/expect").unwrap().body().unwrap(), "gamma\n");
+		assert_eq!(redoc.hunk("a/expect").unwrap().body().unwrap(), "alpha\n");
+
+		std::fs::remove_dir_all(&dir).expect("cleanup");
+	}
+
+	#[test]
+	fn write_back_without_patches_is_a_no_op() {
+		// Point at a path that does not exist: a no-op must not try to write it.
+		let chk = checker(false);
+		let path = std::path::Path::new("/nonexistent/testmark-no-op");
+		assert!(write_back(path, &chk).is_ok());
+	}
+}
diff --git a/crates/testmark/src/files.rs b/crates/testmark/src/files.rs
new file mode 100644
index 0000000..89bcfbd
--- /dev/null
+++ b/crates/testmark/src/files.rs
@@ -0,0 +1,53 @@
+//! File helpers: reading a testmark document from disk.
+//!
+//! Free functions, not methods: `Document` itself has no notion of where its
+//! text came from, and keeping I/O out of the document model keeps that model
+//! pure. (The write-side helper, `write_back`, lives in the `check` module:
+//! writing is only ever the return leg of a check-with-regen cycle.)
+
+use std::fmt;
+use std::path::{Path, PathBuf};
+
+use crate::document::{Document, ParseError};
+
+/// Read and parse a testmark file.
+pub fn read_file(path: &Path) -> Result<Document, ReadFileError> {
+	let text = std::fs::read_to_string(path).map_err(|source| ReadFileError::Io {
+		path: path.to_path_buf(),
+		source,
+	})?;
+	Document::parse(&text).map_err(|source| ReadFileError::Parse {
+		path: path.to_path_buf(),
+		source,
+	})
+}
+
+/// Problems from [`read_file`].
+#[derive(Debug)]
+pub enum ReadFileError {
+	Io {
+		path: PathBuf,
+		source: std::io::Error,
+	},
+	Parse {
+		path: PathBuf,
+		source: ParseError,
+	},
+}
+
+impl fmt::Display for ReadFileError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			ReadFileError::Io {
+				path,
+				source,
+			} => write!(f, "reading {}: {source}", path.display()),
+			ReadFileError::Parse {
+				path,
+				source,
+			} => write!(f, "{}: {source}", path.display()),
+		}
+	}
+}
+
+impl std::error::Error for ReadFileError {}
diff --git a/crates/testmark/src/lib.rs b/crates/testmark/src/lib.rs
index 64b0a22..79e927e 100644
--- a/crates/testmark/src/lib.rs
+++ b/crates/testmark/src/lib.rs
@@ -23,13 +23,20 @@
 //! - [`PatchAccumulator`]: pending hunk rewrites, applied in one pure pass
 //!   by [`Document::patched_all`].  Still pure format manipulation:
 //!   any tool that runs hunks and writes results back can use these two layers.
+//! - [`Checker`]: the testing layer.  Wraps a document and an accumulator,
+//!   and owns the assert-or-regen semantics (the `TESTMARK_REGEN` env var).
+//!
 //! This implementation fails closed where the input looks like testmark but isn't quite:
 //! malformed annotations, annotations without a following fence, unclosed fences,
 //! and duplicate hunk names are all errors rather than silently skipped hunks.
 //! (A silently skipped hunk would mean a silently unexercised test.)
 
+mod check;
 mod document;
+mod files;
 mod patch;
 
+pub use check::{CheckError, Checker, WriteBackError, write_back};
 pub use document::{Document, Hunk, HunkContent, NewContent, ParseError, ParsedTable};
+pub use files::{ReadFileError, read_file};
 pub use patch::{PatchAccumulator, PatchError};