testmark-go

Merge branch 'testexec'

commit 43602084be391995582988e01196de52dcd4fee0 · parent d583ad61f3 · parent c215a37d7c · browse this tree
Eric Myhre, 2021-09-19 17:53:36 UTC

diff --git a/flags.go b/flags.go
new file mode 100644
index 0000000..c80bc9d
--- /dev/null
+++ b/flags.go
@@ -0,0 +1,7 @@
+package testmark
+
+import (
+	"flag"
+)
+
+var Regen = flag.Bool("testmark.regen", false, "Setting this flag hints to systems using testmark tests that fixtures should be regenerated.")
diff --git a/patch.go b/patch.go
index e1342c4..6465ece 100644
--- a/patch.go
+++ b/patch.go
@@ -98,3 +98,21 @@ func appendHunkLines(lines [][]byte, hunkName string, hunkBlockTag string, hunkB
 	lines = append(lines, sigilCodeBlock)
 	return lines
 }
+
+type PatchAccumulator struct {
+	Patches []Hunk
+}
+
+func (pa *PatchAccumulator) AppendPatchIfBodyDiffers(hunk Hunk, newBody []byte) {
+	if !bytes.Equal(hunk.Body, newBody) {
+		hunk.Body = newBody
+		pa.AppendPatch(hunk)
+	}
+}
+
+func (pa *PatchAccumulator) AppendPatch(hunk Hunk) {
+	if pa.Patches == nil {
+		pa.Patches = make([]Hunk, 0)
+	}
+	pa.Patches = append(pa.Patches, hunk)
+}
diff --git a/testexec/defaults.go b/testexec/defaults.go
new file mode 100644
index 0000000..26d3ad6
--- /dev/null
+++ b/testexec/defaults.go
@@ -0,0 +1,27 @@
+package testexec
+
+import (
+	"io"
+	"os/exec"
+	"testing"
+)
+
+func ExecFn_Exec(args []string, stdin io.Reader, stdout, stderr io.Writer) (exitcode int, oshit error) {
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdin = stdin
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err := cmd.Run()
+	// FIXME: get the error code.
+	return -1, err
+}
+
+func ScriptFn_ExecBash(script string, stdin io.Reader, stdout, stderr io.Writer) (exitcode int, oshit error) {
+	return ExecFn_Exec([]string{"bash", "-c", script}, stdin, stdout, stderr)
+}
+
+func defaultAssertFn(t *testing.T, actual, expect string) {
+	if actual != expect {
+		t.Errorf("expected: %q; actual: %q", expect, actual)
+	}
+}
diff --git a/testexec/testexec.go b/testexec/testexec.go
new file mode 100644
index 0000000..8d673fd
--- /dev/null
+++ b/testexec/testexec.go
@@ -0,0 +1,241 @@
+/*
+	testexec is a package that offers some helper functions for testing
+	execution of commands (either with actual os.Exec, or a callback of your defining),
+	and tests them using certain conventions of testmark dir+filename names for the fixture data.
+
+	This package isn't part of testmark's core, nor is it particularly special.
+	It's just some conventions that may be handy.
+*/
+package testexec
+
+import (
+	"bytes"
+	"io"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/warpfork/go-testmark"
+)
+
+// ExecFn is the outline for a callback that can run individual commands in a sequence of commands.
+// An ExecFn can be placed in the Tester config struct to control the behavior of TestSequence.
+//
+// Semantically, an ExecFn is supposed to treat its args slice roughly as `os.StartProcess` treats `argv`:
+// the zeroth element of the slice is the command name (or path), and subsequent slice elements are positional arguments to that command.
+// However, it's a freeform callback: you can do as you wish.
+type ExecFn func(args []string, stdin io.Reader, stdout, stderr io.Writer) (exitcode int, oshit error)
+
+// ScriptFn is the outline for a callback that can run a script.
+// A ScriptFn can be placed in the Tester config struct to control the behavior of TestScript.
+//
+// Semantically, ScriptFn is... quite free to do whatever it wishes.
+// A typical implementation (and the default behavior of this package, if you don't supply a ScriptFn)
+// is to handle the script as if it was a bash shell script (so, piping, env vars, etc, are all possible).
+// However, one could place any kind of script parser and interpreter within this callback.
+type ScriptFn func(script string, stdin io.Reader, stdout, stderr io.Writer) (exitcode int, oshit error)
+
+// FilterFn is the outline for a callback that can be used to normalize some parts of output strings.
+// A common example of this is to strip timestamps back out of log messages for a program that emits logs with timestamps.
+type FilterFn func(line string) (replacement string)
+
+// AssertFn can be used in Tester to specify a better test assertion function.
+// For example, if you use the quicktest package, this is an excellent snippet which will result in good diffs:
+//
+//		func(...) { quicktest.Assert(t, actual, quicktest.CmpEquals(), expect) }
+//
+// The default behavior, if a Tester object doesn't get an AssertFn, is to fall back to a very basic call to `t.Errorf`.
+type AssertFn func(t *testing.T, actual, expect string)
+
+// Tester is a configuration-gathering structure.
+// Each of the `Test*` methods upon it will use these callbacks to define their behavior.
+//
+// All of the fields can be nil, which will result in default behaviors.
+// (A nil ExecFn will result in an OS exec being used (specifically: `ExecFn_Exec`);
+// a nil FilterFn means no filtering will occur;
+// a nil AssertFn means a very basic check using t.Errorf will be used.)
+//
+// The 'Patches' accumulator will be used to gather new fixture data if `testmark.Regen` is true.
+// (If the pointer is nil, a warning will be logged.)
+// It is the user's responsibility to actually apply the patches and save the updated document.
+type Tester struct {
+	ExecFn
+	ScriptFn
+	FilterFn
+	AssertFn
+
+	Patches *testmark.PatchAccumulator
+}
+
+func (tcfg *Tester) init() {
+	if tcfg.ExecFn == nil {
+		tcfg.ExecFn = ExecFn_Exec
+	}
+	if tcfg.ScriptFn == nil {
+		tcfg.ScriptFn = ScriptFn_ExecBash
+	}
+	if tcfg.AssertFn == nil {
+		tcfg.AssertFn = defaultAssertFn
+	}
+}
+
+// TestSequence runs a test based on a "sequence" instruction -- a hunk called "sequence" should have a series of lines,
+// and each line is a command to be executed.
+//
+// The model of execution and model of environment used is very simple:
+// it's just consecutive lines which will be fed one after another to an ExecFn.
+// There is *not* a scripting engine involved (you can't "pipe", use "env vars", use "subcommands",
+// "$?" has no special meaning, etc; there's no shell interpreting those things).
+// (The ExecFn can be as simple or as complex as you want, of course -- but none of these features are provided for you.)
+//
+// The parser that splits up each line of the sequences into the args string slice is primitive.  It's simply whitespace splitting.
+// If you need to test the handling of arguments that involve whitespace, there are two options:
+// One, you may want to use TestScript instead (this will let you parse it yourself, and or just hand off to a shell of some kind which means you can use the shell's quoting rules);
+// or, Two, you can change your sequence hunk name to `sequences.jsonl`, and put a json list on each line, which will be parsed and that list becomes the args.
+//
+// Each DirEnt can also contain several other named entries which will be treated specially.
+// "output" -- if present, will cause the commands to be given a unified stdout and stderr buffer, and it will be checked against this data when done.
+// "stdout" -- if present, works like "output", but only collects stdout.  Cannot be combined with "output".
+// "stderr" -- ditto "stdout", but for (you guessed it) stderr.
+// "exitcode" -- if present, should contains a base-10 number for the expected exit code.  If not present, an exitcode of 0 will be expected.
+//
+// Not every data DirEnt has to contain any of "output", "stdout", "stderr", or "exitcode".
+// Containing none of them means the commands in the sequence will all be run,
+// and the exitcode is expected to be zero from each,
+// and nothing about their stdout nor stderr will be checked.
+// (If you want to mandate that their output is empty, you must create a blank data block to say so explicitly.)
+//
+// If you wish to run some commands, and gather and test (or ignore!) their output as one block,
+// then run additional commands in a subtest, you can use another special path name
+// in the testmark.DirEnt for that: `then-(.*)`.
+// Anything starting with "then-" will cause a new nested `t.Run` block,
+// which will have a name that is the rest of the path name once the "then-" prefix has been stripped.
+// Inside that DirEnt, all the same rules apply again (we'll look for a "sequence" hunk, etc).
+//
+// Note that there are no faculties to expect a specific exitcode from a specific line of the sequence,
+// nor are their faculties to extract outputs from one specific line of the sequence.
+// If you are wanting to do these things, you can use the "then-" feature.
+// (This is probably a nudge towards writing better documentation anyway:
+// if you have a whole series of commands and it's a very specific one that should stand out,
+// you should probably give it separate data blocks for sheer human readability anyway.)
+//
+// There is no faculty for ignoring an exitcode.
+// Surely the systems you're so rigorously ensuring the quality of don't have nondeterministic exit codes;
+// so why would you need to do this?  If it's allowed to be anything but zero, it should be expected to be that; and if it's expected, say so.
+//
+// This test system comes complete with fixture regeneration support.
+// If `testmark.Regen` is true (e.g. you have invoked "go test" with the argument "-testmark.regen"),
+// then instead of making any assertions, this function will accumulate patches
+// in the `Tester.Patches` slice.
+// Regen mode will only update hunks that already exist; it won't add them.
+// As an edge case, note that if that an exitcode hunk is absent, but a nonzero exitcode is encountered,
+// the test will still be failed, even though in patch regen mode most assertions are usually skipped.
+func (tcfg Tester) TestSequence(t *testing.T, data testmark.DirEnt) {
+	t.Helper()
+	tcfg.init()
+
+	sequenceHunk, exists := data.Children["sequence"]
+	if !exists {
+		return
+	}
+	if *testmark.Regen && tcfg.Patches == nil {
+		t.Logf("warning: testmark.regen mode engaged, but there is no patch accumulator available here")
+		t.Skipf("nothing to do if requested to regenerate test fixtures but have nowhere to put data")
+	}
+
+	// Prepare output buffers.
+	var stdout, stderr io.Writer
+	if _, exists := data.Children["output"]; exists {
+		stdout = &bytes.Buffer{}
+		stderr = stdout
+		if _, exists := data.Children["stdout"]; exists {
+			t.Errorf("testexec entry %q shouldn't contain 'stdout' hunk if it also specifies a unified 'output' hunk", data.Name)
+		}
+		if _, exists := data.Children["stderr"]; exists {
+			t.Errorf("testexec entry %q shouldn't contain 'stderr' hunk if it also specifies a unified 'output' hunk", data.Name)
+		}
+	}
+	if _, exists := data.Children["stdout"]; exists {
+		stdout = &bytes.Buffer{}
+	}
+	if _, exists := data.Children["stderr"]; exists {
+		stderr = &bytes.Buffer{}
+	}
+	var exitcode int
+
+	// Loop over the lines in the sequence.
+	lines := bytes.Split(sequenceHunk.Hunk.Body, []byte{'\n'})
+	for _, line := range lines {
+		args := strings.Fields(string(line))
+		if len(args) < 1 {
+			continue
+		}
+
+		var err error
+		exitcode, err = tcfg.ExecFn(args, bytes.NewReader(nil), stdout, stderr)
+		if err != nil {
+			t.Fatalf("execution failed: %s", err)
+		}
+		if exitcode != 0 {
+			break // TODO: it's probably still an error if that happens before the end?
+		}
+	}
+
+	// Okay, comparisons time.
+	// Or, regen time!
+	if ent, exists := data.Children["output"]; exists {
+		bs := stdout.(*bytes.Buffer).Bytes()
+		if *testmark.Regen {
+			tcfg.Patches.AppendPatchIfBodyDiffers(*ent.Hunk, bs)
+		} else {
+			t.Run("check-combined-output", func(t *testing.T) {
+				tcfg.AssertFn(t, string(bs), string(ent.Hunk.Body))
+			})
+		}
+	}
+	if ent, exists := data.Children["stdout"]; exists {
+		bs := stdout.(*bytes.Buffer).Bytes()
+		if *testmark.Regen {
+			tcfg.Patches.AppendPatchIfBodyDiffers(*ent.Hunk, bs)
+		} else {
+			t.Run("check-stdout", func(t *testing.T) {
+				tcfg.AssertFn(t, string(bs), string(ent.Hunk.Body))
+			})
+		}
+	}
+	if ent, exists := data.Children["stderr"]; exists {
+		bs := stderr.(*bytes.Buffer).Bytes()
+		if *testmark.Regen {
+			tcfg.Patches.AppendPatchIfBodyDiffers(*ent.Hunk, bs)
+		} else {
+			t.Run("check-stderr", func(t *testing.T) {
+				tcfg.AssertFn(t, string(bs), string(ent.Hunk.Body))
+			})
+		}
+	}
+	t.Run("check-exitcode", func(t *testing.T) {
+		if ent, exists := data.Children["exitcode"]; exists {
+			if *testmark.Regen {
+				tcfg.Patches.AppendPatchIfBodyDiffers(*ent.Hunk, []byte(strconv.Itoa(exitcode)))
+			} else {
+				tcfg.AssertFn(t, strconv.Itoa(exitcode), strings.TrimSpace(string(ent.Hunk.Body)))
+			}
+		} else {
+			tcfg.AssertFn(t, strconv.Itoa(exitcode), "0")
+		}
+	})
+
+	// TODO: look for "then-*" dirs.
+}
+
+func (tcfg Tester) TestScript(t *testing.T, data testmark.DirEnt) {
+	panic("not yet implemented")
+}
+
+func (tcfg Tester) Test(t *testing.T, data testmark.DirEnt) {
+	panic("not yet implemented")
+}
+
+// Not yet defined how these will nest.  ISTM if you specify one or the other, it shouldn't become willing to switch when it goes deeper into then-trees.
+
+// Not yet defined if these should complain loudly if they _don't_ find something that matches.  I think being able to shrug is useful; otherwise that check will often get offloaded to callers.
diff --git a/write.go b/write.go
index 0a6856e..dc44ffc 100644
--- a/write.go
+++ b/write.go
@@ -2,6 +2,7 @@ package testmark
 
 import (
 	"io"
+	"os"
 	"strings"
 )
 
@@ -27,3 +28,37 @@ func Write(doc *Document, wr io.Writer) (int, error) {
 	}
 	return n, nil
 }
+
+func WriteFile(doc *Document, filename string) error {
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = Write(doc, f)
+	return err
+}
+
+func WriteWithPatches(doc *Document, wr io.Writer, patches ...Hunk) (int, error) {
+	if len(patches) == 0 {
+		return 0, nil
+	}
+	doc = Patch(doc, patches...)
+	return Write(doc, wr)
+}
+
+func WriteFileWithPatches(doc *Document, filename string, patches ...Hunk) error {
+	if len(patches) == 0 {
+		return nil
+	}
+	doc = Patch(doc, patches...)
+	return WriteFile(doc, filename)
+}
+
+func (pa PatchAccumulator) WriteWithPatches(doc *Document, wr io.Writer) (int, error) {
+	return WriteWithPatches(doc, wr, pa.Patches...)
+}
+
+func (pa PatchAccumulator) WriteFileWithPatches(doc *Document, filename string) error {
+	return WriteFileWithPatches(doc, filename, pa.Patches...)
+}