testmark-go

Add recursion flexibility

commit 4e855bce0a6121dc20bb2843656f1f8492272904 · parent 095106dc5d · browse this tree
Calvin Behling, 2022-11-13 18:52:34 UTC

- Adds tests to output that will show up as skipped.
This should make the behavior of testexec easier understand and debug
using go test -v.
- Adds a RecursionFn to testexec to give users the ability to control recursion.
- Adds a Path variable to DirEnt structs which contains its full path.

Overall default behavior is unchanged.

diff --git a/go.mod b/go.mod
index dc56ee0..071e0b4 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
 module github.com/warpfork/go-testmark
 
 go 1.16
+
+require github.com/frankban/quicktest v1.14.3 // indirect
diff --git a/index.go b/index.go
index 6d3463d..5783e43 100644
--- a/index.go
+++ b/index.go
@@ -16,27 +16,28 @@ import (
 func (doc *Document) BuildDirIndex() {
 	doc.DirEnt = &DirEnt{}
 	for _, hunk := range doc.DataHunks {
-		doc.DirEnt.fill(strings.Split(hunk.Name, "/"), hunk.Hunk)
+		doc.DirEnt.fill(strings.Split(hunk.Name, "/"), 0, hunk.Hunk)
 	}
 }
 
-func (dirent *DirEnt) fill(pathSegs []string, hunk Hunk) {
-	if len(pathSegs) == 0 {
+func (dirent *DirEnt) fill(pathSegs []string, pathIdx int, hunk Hunk) {
+	if pathIdx >= len(pathSegs) {
 		dirent.Hunk = &hunk
 		return
 	}
 	if dirent.Children == nil {
 		dirent.Children = make(map[string]*DirEnt)
 	}
-	if next, exists := dirent.Children[pathSegs[0]]; exists {
-		next.fill(pathSegs[1:], hunk)
-	} else {
-		l := len(dirent.ChildrenList)
-		child := DirEnt{
-			Name: pathSegs[0],
-		}
-		dirent.ChildrenList = append(dirent.ChildrenList, &child)
-		dirent.Children[pathSegs[0]] = &child
-		dirent.ChildrenList[l].fill(pathSegs[1:], hunk)
+	if next, exists := dirent.Children[pathSegs[pathIdx]]; exists {
+		next.fill(pathSegs, pathIdx+1, hunk)
+		return
 	}
+
+	l := len(dirent.ChildrenList)
+	dirent.ChildrenList = append(dirent.ChildrenList, DirEnt{
+		Name: pathSegs[pathIdx],
+		Path: strings.Join(pathSegs[:pathIdx+1], "/"),
+	})
+	dirent.Children[pathSegs[pathIdx]] = &dirent.ChildrenList[l]
+	dirent.ChildrenList[l].fill(pathSegs, pathIdx+1, hunk)
 }
diff --git a/testexec/defaults.go b/testexec/defaults.go
index 3ecad65..2403593 100644
--- a/testexec/defaults.go
+++ b/testexec/defaults.go
@@ -1,10 +1,14 @@
 package testexec
 
 import (
+	"fmt"
 	"io"
 	"os/exec"
+	"strings"
 	"syscall"
 	"testing"
+
+	"github.com/warpfork/go-testmark"
 )
 
 func ExecFn_Exec(args []string, stdin io.Reader, stdout, stderr io.Writer) (exitcode int, oshit error) {
@@ -46,3 +50,28 @@ func defaultAssertFn(t *testing.T, actual, expect string) {
 		t.Errorf("expected: %q; actual: %q", expect, actual)
 	}
 }
+
+// Hash Table of all the "special" nodes used by testmark.
+var DefaultLeafNodes = map[string]struct{}{
+	"exitcode": {},
+	"stderr":   {},
+	"stdout":   {},
+	"output":   {},
+	"input":    {},
+	"sequence": {},
+	"script":   {},
+	"fs":       {},
+}
+
+func RecursionFn_Then(t *testing.T, dir testmark.DirEnt) error {
+	t.Helper()
+	if _, exists := DefaultLeafNodes[dir.Name]; exists {
+		t.Logf("will not recurse into special leaf node %q", dir.Name)
+		return IgnoreRecursion
+	}
+	if len(dir.Name) > 5 && strings.HasPrefix(dir.Name, "then-") {
+		return nil
+	}
+	t.Logf("skipping dir recursion: %q does not begin with %q ", dir.Name, "then-")
+	return fmt.Errorf("%q does not begin with %q: %w", dir.Name, "then-", SkipRecursion)
+}
diff --git a/testexec/selfexercise.md b/testexec/selfexercise.md
index 67d4842..1040c6c 100644
--- a/testexec/selfexercise.md
+++ b/testexec/selfexercise.md
@@ -99,3 +99,23 @@ cat - | sed 's/ is/ was/'  | sed s/will/should/
 ```
 this was stdin and should be echoed
 ```
+
+---
+
+The default recursion function won't reach everything if you don't formulate your paths correctly.
+
+[testmark]:# (bad/script)
+```
+  # no-op
+```
+
+[testmark]:# (bad/then-missing-script/then-another-thing/script)
+```
+  # no-op, this will not be reached because it missing executable blocks in parent steps
+```
+
+[testmark]:# (bad/not-a-then-statement/script)
+```
+  # no-op, this script will not run because the subdirectory does not begin with "then-"
+```
+
diff --git a/testexec/testexec.go b/testexec/testexec.go
index 8006d83..9c2e30d 100644
--- a/testexec/testexec.go
+++ b/testexec/testexec.go
@@ -10,6 +10,7 @@ package testexec
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"io/ioutil"
 	"os"
@@ -50,6 +51,18 @@ type FilterFn func(line string) (replacement string)
 // The default behavior, if a Tester object doesn't get an AssertFn, is to fall back to a very basic call to `t.Errorf`.
 type AssertFn func(t *testing.T, actual, expect string)
 
+var (
+	// SkipRecursion can be returned by a RecursionFn for paths that should show up in the test as skipped
+	SkipRecursion = errors.New("skip this directory")
+	// IgnoreRecursion can be returned by a RecursionFn for paths that should not run a new test at all
+	IgnoreRecursion = errors.New("ignore this directory")
+)
+
+// RecursionFn can be used in Tester to specify a recursion pattern
+// The default recursion pattern will recurse whenever it finds "then-"
+// Returning a non-nil error will cause the test for the directory to fail unless the error is a SkipRecursion or IgnoreRecursion error
+type RecursionFn func(*testing.T, testmark.DirEnt) error
+
 // Tester is a configuration-gathering structure.
 // Each of the `Test*` methods upon it will use these callbacks to define their behavior.
 //
@@ -66,6 +79,7 @@ type Tester struct {
 	ScriptFn
 	FilterFn
 	AssertFn
+	RecursionFn
 
 	Patches *testmark.PatchAccumulator
 }
@@ -161,11 +175,10 @@ func (tcfg Tester) test(t *testing.T, data *testmark.DirEnt, allowExec, allowScr
 	sequenceHunk, sequenceMode := data.Children["sequence"]
 	scriptHunk, scriptMode := data.Children["script"]
 	if !sequenceMode && !scriptMode {
-		return
+		t.Skipf("dir %q does not contain a 'script' or 'sequence' hunk and will be skipped", data.Name)
 	}
 	if sequenceMode && scriptMode {
-		t.Logf("warning: dir %q contained both a 'script' and a 'sequence' hunk, which is nonsensical", data.Name)
-		t.SkipNow()
+		t.Skipf("warning: dir %q contained both a 'script' and a 'sequence' hunk, which is nonsensical", data.Name)
 	}
 	if sequenceMode && !allowExec {
 		t.Skipf("found sequence hunk but the test framework was invoked without permission to run those")
@@ -299,25 +312,37 @@ func (tcfg Tester) test(t *testing.T, data *testmark.DirEnt, allowExec, allowScr
 		}
 	})
 
-	// Look for "then-*" dirs.
-	//  If we're already failed -- make the run block, but skip it.
-	//  If we're going to procede: make a new tempdir, copy the contents, and then procede by recursing.
+	tcfg.recurse(t, data, allowExec, allowScript, dir)
+}
+
+func (tcfg Tester) recurse(t *testing.T, data testmark.DirEnt, allowExec bool, allowScript bool, parentTmpDir string) {
+	if tcfg.RecursionFn == nil {
+		tcfg.RecursionFn = RecursionFn_Then
+	}
 	for _, child := range data.ChildrenList {
-		if len(child.Name) > 5 && strings.HasPrefix(child.Name, "then-") {
-			alreadyFailed := t.Failed()
-			t.Run(child.Name, func(t *testing.T) {
-				if alreadyFailed {
-					t.Skipf("parent commands failed, so while more commands are specified, testing them is not meaningful")
-				}
-				tcfg.test(t, child, allowExec, allowScript, dir)
-			})
+		recurseErr := tcfg.RecursionFn(t, child)
+		if errors.Is(recurseErr, IgnoreRecursion) {
+			continue
 		}
+		alreadyFailed := t.Failed()
+		t.Run(child.Name, func(t *testing.T) {
+			if alreadyFailed {
+				t.Skipf("parent commands failed, so while more commands are specified, testing them is not meaningful")
+			}
+			if errors.Is(recurseErr, SkipRecursion) {
+				t.Skip(recurseErr)
+			}
+			if recurseErr != nil {
+				t.Fatal(recurseErr)
+			}
+			tcfg.test(t, child, allowExec, allowScript, parentTmpDir)
+		})
 	}
-
 }
 
 func (tcfg Tester) doSequence(t *testing.T, hunk *testmark.Hunk, stdin io.Reader, stdout, stderr io.Writer) (exitcode int) {
 	t.Helper()
+	t.Logf("running sequence: %q", hunk.Name)
 	// Loop over the lines in the sequence.
 	lines := bytes.Split(hunk.Body, []byte{'\n'})
 	for _, line := range lines {
@@ -340,6 +365,7 @@ func (tcfg Tester) doSequence(t *testing.T, hunk *testmark.Hunk, stdin io.Reader
 
 func (tcfg Tester) doScript(t *testing.T, hunk *testmark.Hunk, stdin io.Reader, stdout, stderr io.Writer) (exitcode int) {
 	t.Helper()
+	t.Logf("running script: %q", hunk.Name)
 	var err error
 	exitcode, err = tcfg.ScriptFn(string(hunk.Body), stdin, stdout, stderr)
 	if err != nil {
diff --git a/testexec/testexec_test.go b/testexec/testexec_test.go
index 0701f4a..ae1ed70 100644
--- a/testexec/testexec_test.go
+++ b/testexec/testexec_test.go
@@ -1,13 +1,16 @@
 package testexec_test
 
 import (
+	"sort"
 	"testing"
 
 	"github.com/warpfork/go-testmark"
 	"github.com/warpfork/go-testmark/testexec"
+
+	qt "github.com/frankban/quicktest"
 )
 
-func Test(t *testing.T) {
+func TestSelf(t *testing.T) {
 	filename := "selfexercise.md"
 	doc, err := testmark.ReadFile(filename)
 	if err != nil {
@@ -26,3 +29,126 @@ func Test(t *testing.T) {
 	}
 	patches.WriteFileWithPatches(doc, filename)
 }
+
+// TestRecursion tests edge cases around which nodes are traversed via recursion
+func TestRecursion(t *testing.T) {
+	rtested := map[string]int{}
+	rcalled := map[string]int{}
+	sum := func(m map[string]int) int {
+		result := 0
+		for _, v := range m {
+			result += v
+		}
+		return result
+	}
+
+	rfn := func(t *testing.T, dir testmark.DirEnt) error {
+		rtested[dir.Path]++
+		result := testexec.RecursionFn_Then(t, dir)
+		if result == nil {
+			rcalled[dir.Path]++
+		}
+		return result
+	}
+
+	filename := "selfexercise.md"
+	doc, err := testmark.ReadFile(filename)
+	if err != nil {
+		t.Fatalf("spec file parse failed?!: %s", err)
+	}
+
+	doc.BuildDirIndex()
+
+	patches := testmark.PatchAccumulator{}
+	for _, dir := range doc.DirEnt.ChildrenList {
+		rtested[dir.Path]++
+		t.Run(dir.Name, func(t *testing.T) {
+			rcalled[dir.Path]++
+			test := testexec.Tester{
+				Patches:     &patches,
+				RecursionFn: rfn,
+			}
+			test.Test(t, dir)
+		})
+	}
+	patches.WriteFileWithPatches(doc, filename)
+
+	pathList := collectPaths(*doc.DirEnt)
+	sort.Strings(pathList)
+	t.Log("[idx]: [test count]-[recursion count]: [path]")
+	for idx, path := range pathList {
+		check := rtested[path]
+		recurse := rcalled[path]
+		t.Logf("%02d: %d-%d: %s", idx, check, recurse, path)
+	}
+
+	qt.Assert(t, rtested[""], qt.Equals, 0)
+	qt.Assert(t, rcalled[""], qt.Equals, 0)
+	qt.Assert(t, rtested["bad"], qt.Equals, 1)
+	qt.Assert(t, rcalled["bad"], qt.Equals, 1)
+	qt.Assert(t, rtested["bad/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["bad/script"], qt.Equals, 0)
+	qt.Assert(t, rtested["bad/then-missing-script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["bad/then-missing-script"], qt.Equals, 1)
+	qt.Assert(t, rtested["bad/then-missing-script/then-another-thing"], qt.Equals, 0)
+	qt.Assert(t, rcalled["bad/then-missing-script/then-another-thing"], qt.Equals, 0)
+	qt.Assert(t, rtested["bad/then-missing-script/then-another-thing/script"], qt.Equals, 0)
+	qt.Assert(t, rcalled["bad/then-missing-script/then-another-thing/script"], qt.Equals, 0)
+	qt.Assert(t, rtested["bad/not-a-then-statement"], qt.Equals, 1)
+	qt.Assert(t, rcalled["bad/not-a-then-statement"], qt.Equals, 0)
+	qt.Assert(t, rtested["bad/not-a-then-statement/script"], qt.Equals, 0)
+	qt.Assert(t, rcalled["bad/not-a-then-statement/script"], qt.Equals, 0)
+
+	qt.Assert(t, rtested["using-stdin"], qt.Equals, 1)
+	qt.Assert(t, rcalled["using-stdin"], qt.Equals, 1)
+	qt.Assert(t, rtested["using-stdin/input"], qt.Equals, 1)
+	qt.Assert(t, rcalled["using-stdin/input"], qt.Equals, 0)
+	qt.Assert(t, rtested["using-stdin/output"], qt.Equals, 1)
+	qt.Assert(t, rcalled["using-stdin/output"], qt.Equals, 0)
+	qt.Assert(t, rtested["using-stdin/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["using-stdin/script"], qt.Equals, 0)
+
+	qt.Assert(t, rtested["whee"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee"], qt.Equals, 1)
+	qt.Assert(t, rtested["whee/fs"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/fs"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/fs/a"], qt.Equals, 0)
+	qt.Assert(t, rcalled["whee/fs/a"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/output"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/output"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/script"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-more-files"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-more-files"], qt.Equals, 1)
+	qt.Assert(t, rtested["whee/then-more-files/fs"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-more-files/fs"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-more-files/fs/b"], qt.Equals, 0)
+	qt.Assert(t, rcalled["whee/then-more-files/fs/b"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-more-files/output"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-more-files/output"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-more-files/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-more-files/script"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-touching-files"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files"], qt.Equals, 1)
+	qt.Assert(t, rtested["whee/then-touching-files/output"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files/output"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-touching-files/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files/script"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-touching-files/then-subtesting-again"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files/then-subtesting-again"], qt.Equals, 1)
+	qt.Assert(t, rtested["whee/then-touching-files/then-subtesting-again/output"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files/then-subtesting-again/output"], qt.Equals, 0)
+	qt.Assert(t, rtested["whee/then-touching-files/then-subtesting-again/script"], qt.Equals, 1)
+	qt.Assert(t, rcalled["whee/then-touching-files/then-subtesting-again/script"], qt.Equals, 0)
+
+	qt.Assert(t, sum(rtested), qt.Equals, 22)
+	qt.Assert(t, sum(rcalled), qt.Equals, 7)
+}
+
+func collectPaths(d testmark.DirEnt) []string {
+	result := []string{d.Path}
+	for _, c := range d.ChildrenList {
+		result = append(result, collectPaths(c)...)
+	}
+	return result
+}
diff --git a/testmark.go b/testmark.go
index 4113279..5603c0d 100644
--- a/testmark.go
+++ b/testmark.go
@@ -67,6 +67,9 @@ type DirEnt struct {
 	// (Note that if there's a Hunk in this DirEnt, its name may be different -- it still has the *full* path name.)
 	Name string
 
+	// The fullpath
+	Path string
+
 	// A hunk, or nil.
 	Hunk *Hunk