testmark-go

FS implementation (#11)

commit 8c1550b9059ed5c0437f5085cffa4dc9a752d8f4 · parent 5cd7a73883 · browse this tree
CJB, 2022-11-27 21:02:13 UTC

diff --git a/fs/fs.go b/fs/fs.go
new file mode 100644
index 0000000..b95d472
--- /dev/null
+++ b/fs/fs.go
@@ -0,0 +1,232 @@
+// EXPERIMENTAL
+// The testmark/fs package is an implementation of the io/fs interface for testmark documents.
+// Some behavior of testmark and io/fs is incompatible. Known issues include handling of
+//  relative paths and overlapping regular files with directories.
+//
+// This package isn't part of testmark's core and breaking changes may be frequent.
+package fs
+
+import (
+	"bytes"
+	"io/fs"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/warpfork/go-testmark"
+)
+
+// We don't implement writing to hunks through this interface so everything is read-only
+const defaultFileMode fs.FileMode = 0444
+
+// File is a representation of DirEnt used when working with Documents as an fs.FS
+// Generally this is done by calling Document.Open or using a function from the fs package.
+type File struct {
+	// buffer contains the hunk data after opening
+	// a nil buffer implies that the file is closed
+	buffer *bytes.Buffer
+	stat   fileStat
+	// childIdx tracks index of children for readdir
+	childIdx int
+	// children sorted by name
+	childrenSorted []string
+	children       map[string]*testmark.DirEnt
+}
+
+// Name returns the name of the file (or subdirectory) described by the entry.
+// This name is only the final element of the path (the base name), not the entire path.
+// For example, Name would return "hello.go" not "home/gopher/hello.go".
+func (f *File) Name() string {
+	return f.stat.name
+}
+
+// Info is equivalent to Stat
+func (f *File) Info() (fs.FileInfo, error) {
+	return f.stat, nil
+}
+
+// Stat returns a fs.FileInfo for the file
+func (f *File) Stat() (fs.FileInfo, error) {
+	return f.stat, nil
+}
+
+// IsDir reports whether the entry describes a directory.
+func (f *File) IsDir() bool {
+	return f.stat.IsDir()
+}
+
+// Type returns the type bits for the entry.
+// The type bits are a subset of the usual FileMode bits, those returned by the FileMode.Type method.
+func (f *File) Type() fs.FileMode {
+	return f.stat.mode.Type()
+}
+
+// Read reads up to len(b) bytes from the File and stores them in b. It returns
+// the number of bytes read and any error encountered. At end of file, Read
+// returns 0, io.EOF.
+// May return fs.ErrClosed
+func (f *File) Read(b []byte) (int, error) {
+	if f.buffer == nil {
+		return 0, fs.ErrClosed
+	}
+	return f.buffer.Read(b)
+}
+
+// ReadDir will return a []*File as an []fs.DirEntry
+// Returned entries will be sorted by filename as required by fs.ReadDir
+// If n < 0, ReadDir will return all remaining entries.
+func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
+	if len(f.children) == 0 {
+		return []fs.DirEntry{}, nil
+	}
+	start := f.childIdx
+	end := f.childIdx + n
+	if end >= len(f.children) || n < 0 {
+		end = len(f.children)
+	}
+	defer func() { f.childIdx = end }()
+	names := f.childrenSorted[start:end]
+	return f.readDir(names...)
+}
+
+// readDir takes the list of children names and returns an []fs.DirEntry in the same order.
+// Will return an fs.PathError if a child name does not exist.
+func (f *File) readDir(children ...string) ([]fs.DirEntry, error) {
+	result := make([]fs.DirEntry, 0, len(children))
+	for _, name := range children {
+		child, exists := f.children[name]
+		if !exists {
+			return []fs.DirEntry{}, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
+		}
+		result = append(result, file(child))
+	}
+	return result, nil
+}
+
+// Close will set the underlying buffer to nil
+// If the buffer is already nil then Close will return an fs.ErrClosed
+func (f *File) Close() error {
+	if f.buffer == nil {
+		return fs.ErrClosed
+	}
+	f.buffer = nil
+	return nil
+}
+
+// fileStat stores data as an implementation of fs.FileInfo
+type fileStat struct {
+	name string
+	size int64
+	mode fs.FileMode
+	sys  interface{}
+}
+
+// IsDir will be true if a DirEnt has children
+func (s fileStat) IsDir() bool {
+	return s.mode.IsDir()
+}
+
+// ModTime is meaningless. Testmark hunks don't have a modtime
+// ModTime will always return a time.Time zero value
+func (fileStat) ModTime() time.Time {
+	return time.Time{}
+}
+
+// Mode returns the FileMode for the file.
+// The filemode will have the fs.ModeDir bit enabled if the File contains references to children
+func (s fileStat) Mode() fs.FileMode {
+	return s.mode
+}
+
+// Name returns the base name of the file
+func (s fileStat) Name() string {
+	return s.name
+}
+
+// Size returns the size of the underlying hunk when the file was opened
+// If no hunk exists for the file path then size will be the number of children for the directory.
+func (s fileStat) Size() int64 {
+	return s.size
+}
+
+// Sys returns the _thing_ from whence this file came.
+// Generally that's the *DirEnt
+func (s fileStat) Sys() interface{} {
+	return s.sys
+}
+
+type fsimpl testmark.Document
+
+func DocFs(doc *testmark.Document) fs.FS {
+	return (*fsimpl)(doc)
+}
+
+// Open opens the named hunk or a directory path.
+// Open does not follow relative paths such as .. or .
+//
+// When Open returns an error, it should be of type *PathError
+// with the Op field set to "open", the Path field set to name,
+// and the Err field describing the problem.
+//
+// Open does NOT follow conventions for fs.ValidPath.
+// Opening an empty path will return the root directory for the document.
+// This is different than the fs.ValidPath special case of using "." as the root path.
+// The testmark document treats "." and ".." the same as any other character.
+func (f *fsimpl) Open(name string) (fs.File, error) {
+	doc := (*testmark.Document)(f)
+	if doc.DirEnt == nil {
+		doc.BuildDirIndex()
+	}
+	if name == "" {
+		return file(doc.DirEnt), nil
+	}
+	ent := findDir(doc.DirEnt, strings.Split(name, testmark.HunkPathSeparator)...)
+	if ent == nil {
+		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+	}
+	return file(ent), nil
+}
+
+// file returns the File representation of a DirEnt.
+func file(d *testmark.DirEnt) *File {
+	size := int64(len(d.Children))
+	buf := bytes.NewBuffer([]byte{})
+	if d.Hunk != nil {
+		buf = bytes.NewBuffer(d.Hunk.Body)
+		size = int64(len(d.Hunk.Body))
+	}
+	childrenSorted := make([]string, 0, len(d.Children))
+	for name := range d.Children {
+		childrenSorted = append(childrenSorted, name)
+	}
+	sort.Strings(childrenSorted)
+	mode := defaultFileMode
+	if len(d.Children) > 0 {
+		mode = mode | fs.ModeDir
+	}
+	return &File{
+		buffer:         buf,
+		childrenSorted: childrenSorted,
+		children:       d.Children,
+		stat: fileStat{
+			name: d.Name,
+			mode: mode,
+			sys:  d,
+			// size is generally the number of directory entires or the length of the file in bytes.
+			// We have both, you're on your own.
+			size: size,
+		},
+	}
+}
+
+// findDir will traverse down a DirEnt by each path split and return a DirEnt for that path if it exists.
+// Otherwise, findDir will return nil.
+func findDir(dir *testmark.DirEnt, pathsplits ...string) *testmark.DirEnt {
+	if len(pathsplits) == 0 {
+		return dir
+	}
+	if child, exists := dir.Children[pathsplits[0]]; exists {
+		return findDir(child, pathsplits[1:]...)
+	}
+	return nil
+}
diff --git a/fs/fs_test.go b/fs/fs_test.go
new file mode 100644
index 0000000..0179207
--- /dev/null
+++ b/fs/fs_test.go
@@ -0,0 +1,367 @@
+package fs_test
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"path/filepath"
+	"testing"
+
+	"github.com/warpfork/go-testmark"
+	tmfs "github.com/warpfork/go-testmark/fs"
+)
+
+// Assert the implementation of various interfaces in the "fs" package
+var (
+	_ fs.DirEntry    = &tmfs.File{}
+	_ fs.File        = &tmfs.File{}
+	_ fs.ReadDirFile = &tmfs.File{}
+)
+
+// TestFS tests some basic assertions about the *Document implementation of the fs.FS interface
+func TestFS(t *testing.T) {
+	testdata, err := filepath.Abs("../testdata")
+	if err != nil {
+		panic(err)
+	}
+
+	doc, err := testmark.ReadFile(filepath.Join(testdata, "example.md"))
+	if err != nil {
+		panic(err)
+	}
+	dfs := tmfs.DocFs(doc)
+
+	t.Run("open dot path", func(t *testing.T) {
+		f, err := dfs.Open(".")
+		pathErr := new(fs.PathError)
+		if !errors.As(err, &pathErr) {
+			t.Errorf("expected open to return an fs.PathError: %v", err)
+		}
+		assertString(t, pathErr.Op, "open")
+		assertString(t, pathErr.Path, ".")
+		if !errors.Is(err, fs.ErrNotExist) {
+			t.Errorf("expected an fs.ErrNotExist: %v", err)
+		}
+		if f != nil {
+			t.Errorf("expected opened file to be nil")
+		}
+	})
+
+	t.Run("open empty path", func(t *testing.T) {
+		f, err := dfs.Open("")
+		if err != nil {
+			panic(err)
+		}
+		s, err := f.Stat()
+		if err != nil {
+			panic(err)
+		}
+		if !s.IsDir() {
+			t.Errorf("expected file to be a directory")
+		}
+		assertString(t, s.Name(), "")
+	})
+}
+
+func TestFSGlob(t *testing.T) {
+	testdata, err := filepath.Abs("../testdata")
+	if err != nil {
+		panic(err)
+	}
+
+	doc, err := testmark.ReadFile(filepath.Join(testdata, "exampleWithDirs.md"))
+	if err != nil {
+		panic(err)
+	}
+	dfs := tmfs.DocFs(doc)
+
+	matches, err := fs.Glob(dfs, "one/t*")
+	if err != nil {
+		panic(err)
+	}
+	assertStrings(t, matches, "one/three", "one/two")
+}
+
+func TestFSReadFile(t *testing.T) {
+	testdata, err := filepath.Abs("../testdata")
+	if err != nil {
+		panic(err)
+	}
+
+	doc, err := testmark.ReadFile(filepath.Join(testdata, "exampleWithDirs.md"))
+	if err != nil {
+		panic(err)
+	}
+	dfs := tmfs.DocFs(doc)
+
+	data, err := fs.ReadFile(dfs, "one")
+	if err != nil {
+		panic(err)
+	}
+
+	assertString(t, string(data), "baz\n")
+}
+
+func ExampleWalkDir() {
+	testdata, _ := filepath.Abs("../testdata")
+	doc, _ := testmark.ReadFile(filepath.Join(testdata, "example.md"))
+	dfs := tmfs.DocFs(doc)
+
+	counter := 0
+	fs.WalkDir(dfs, "", func(path string, dir fs.DirEntry, err error) error {
+		fmt.Printf("%d: %q\n", counter, path)
+		counter++
+		return nil
+	})
+	// Output:
+	// 0: ""
+	// 1: "cannot-describe-no-linebreak"
+	// 2: "more-data"
+	// 3: "this-is-the-data-name"
+}
+
+func ExampleRead() {
+	testdata, _ := filepath.Abs("../testdata")
+	doc, _ := testmark.ReadFile(filepath.Join(testdata, "example.md"))
+	dfs := tmfs.DocFs(doc)
+	f, _ := dfs.Open("more-data")
+
+	content, _ := io.ReadAll(f)
+	fmt.Print(string(content))
+	// Output:
+	// func OtherMarkdownParsers() (shouldHighlight bool) {
+	// 	return true
+	// }
+}
+
+func ExampleReadDirFile() {
+	testdata, _ := filepath.Abs("../testdata")
+	doc, _ := testmark.ReadFile(filepath.Join(testdata, "example.md"))
+	dfs := tmfs.DocFs(doc)
+
+	f, _ := dfs.Open("")
+	fr := f.(fs.ReadDirFile)
+	dirs, _ := fr.ReadDir(-1)
+	for i, d := range dirs {
+		fmt.Printf("%d: %q\n", i, d.Name())
+	}
+	// Output:
+	// 0: "cannot-describe-no-linebreak"
+	// 1: "more-data"
+	// 2: "this-is-the-data-name"
+}
+
+// Generally true as of this writing
+// A directory will return true on IsDir
+// A directory that is not a file will have a size equal to the number of children
+// A path that is a file will have a size equal to it's buffer length regardless of directory status
+func ExampleIsItAFileOrADirectory() {
+	testdata, _ := filepath.Abs("../testdata")
+	doc, _ := testmark.ReadFile(filepath.Join(testdata, "exampleWithDirs.md"))
+	dfs := tmfs.DocFs(doc)
+	{
+		f, _ := dfs.Open("")
+		stat, _ := f.Stat()
+		fmt.Printf("the root dir with %d children is not a file: %q,%d,%t\n", stat.Size(), stat.Name(), stat.Size(), stat.IsDir())
+	}
+	{
+		f, _ := dfs.Open("one")
+		stat, _ := f.Stat()
+		fmt.Printf("this path is a dir AND a regular file: %q,%d,%t\n", stat.Name(), stat.Size(), stat.IsDir())
+	}
+	{
+		f, _ := dfs.Open("one/four/bang")
+		stat, _ := f.Stat()
+		fmt.Printf("this path is a file but NOT a dir: %q,%d,%t\n", stat.Name(), stat.Size(), stat.IsDir())
+	}
+	// Output:
+	// the root dir with 2 children is not a file: "",2,true
+	// this path is a dir AND a regular file: "one",4,true
+	// this path is a file but NOT a dir: "bang",4,false
+}
+
+func ExampleConvertFileToDirEnt() {
+	testdata, _ := filepath.Abs("../testdata")
+	doc, _ := testmark.ReadFile(filepath.Join(testdata, "example.md"))
+	dfs := tmfs.DocFs(doc)
+
+	f, _ := dfs.Open("more-data")
+	stat, _ := f.Stat()
+	ent := stat.Sys().(*testmark.DirEnt)
+	fmt.Print(string(ent.Hunk.Body))
+	// Output:
+	// func OtherMarkdownParsers() (shouldHighlight bool) {
+	// 	return true
+	// }
+}
+
+// TestWalkDocument tests the implementation of fs.WalkDir against a Document
+func TestFSWalkDocument(t *testing.T) {
+	testdata, err := filepath.Abs("../testdata")
+	if err != nil {
+		panic(err)
+	}
+
+	doc, err := testmark.ReadFile(filepath.Join(testdata, "exampleWithDirs.md"))
+	if err != nil {
+		panic(err)
+	}
+	dfs := tmfs.DocFs(doc)
+
+	type expectT struct {
+		path        string
+		contents    string
+		numChildren int
+	}
+	expected := []expectT{
+		//path, isDir
+		{"", "", 2},
+		{"one", "baz\n", 3}, // both dir and file
+		{"one/four", "", 1},
+		{"one/four/bang", "mop\n", 0},
+		{"one/three", "bar\n", 0},
+		{"one/two", "foo\n", 0},
+		{"really", "", 1},
+		{"really/deep", "", 1},
+		{"really/deep/dirs", "", 1},
+		{"really/deep/dirs/wow", "zot\n", 0},
+	}
+	maxPathLen := 0
+	for _, v := range expected {
+		if maxPathLen < len(v.path) {
+			maxPathLen = len(v.path)
+		}
+	}
+	orderIdx := 0
+
+	mark := func(b bool) string {
+		if b {
+			return "✔"
+		}
+		t.Fail()
+		return "✖"
+	}
+	err = fs.WalkDir(dfs, "", func(path string, dir fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		order := expected[orderIdx]
+		f, err := dfs.Open(path)
+		if err != nil {
+			panic(err)
+		}
+		defer func() {
+			if err := f.Close(); err != nil {
+				panic(err)
+			}
+		}()
+		content, err := io.ReadAll(f)
+		if err != nil {
+			panic(err)
+		}
+
+		fdir := f.(fs.ReadDirFile)
+		children, err := fdir.ReadDir(-1)
+		if err != nil {
+			panic(err)
+		}
+
+		t.Logf("%2d:  path: %s %-*q  children: %s%s %5t,%d  contents: %s %q",
+			orderIdx,
+			mark(order.path == path),
+			maxPathLen+2, // path + quotes. Any path that exceeds the expected length will screw up formatting but that's fine.
+			path,
+			mark(order.numChildren == len(children)),
+			mark(order.numChildren > 0 == dir.IsDir()),
+			dir.IsDir(),
+			len(children),
+			mark(order.contents == string(content)),
+			content,
+		)
+		orderIdx++
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+	if orderIdx != len(expected) {
+		t.Errorf("didn't iterate through all entries")
+	}
+}
+
+// This is a weird edge of object filesystems where pseudo-dirs and files can overlap.
+// We don't really have a great way of handling this. Just, some files are also directories. The end.
+func TestFSOpenFileDir(t *testing.T) {
+	testdata, err := filepath.Abs("../testdata")
+	if err != nil {
+		panic(err)
+	}
+
+	doc, err := testmark.ReadFile(filepath.Join(testdata, "exampleWithDirs.md"))
+	if err != nil {
+		panic(err)
+	}
+	dfs := tmfs.DocFs(doc)
+
+	f, err := dfs.Open("one")
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	assertString(t, string(data), "baz\n")
+
+	rf := f.(fs.ReadDirFile)
+	dirs, err := rf.ReadDir(-1)
+	if err != nil {
+		panic(err)
+	}
+	dirNames := make([]string, 0, len(dirs))
+	for _, d := range dirs {
+		dirNames = append(dirNames, d.Name())
+	}
+
+	assertStrings(t, dirNames, "four", "three", "two")
+
+	stat, err := f.Stat()
+	if err != nil {
+		panic(err)
+	}
+	if !stat.IsDir() {
+		t.Errorf("file should be a directory")
+	}
+	if stat.Name() != "one" {
+		t.Errorf("file name expected to be %q but got %q", "one", stat.Name())
+	}
+	if stat.Size() != int64(len(data)) {
+		t.Errorf("file size of %d expected to be equal to data length of %d", stat.Size(), len(data))
+	}
+}
+
+func assertStrings(t testing.TB, actual []string, expected ...string) {
+	t.Helper()
+	sharedLen := len(actual)
+	if sharedLen > len(expected) {
+		sharedLen = len(expected)
+	}
+	for i := 0; i < sharedLen; i++ {
+		if actual[i] != expected[i] {
+			t.Errorf("Expected actual[%d] to be %q but got %q", i, expected[i], actual[i])
+		}
+	}
+	if len(actual) != len(expected) {
+		t.Errorf("Expected len(actual)=%d to equal len(expected)=%d", len(actual), len(expected))
+	}
+}
+
+func assertString(t testing.TB, actual string, expected string) {
+	t.Helper()
+	if actual != expected {
+		t.Errorf("expected %q to equal %q", actual, expected)
+	}
+}
diff --git a/index.go b/index.go
index 6c56b29..1adb8b1 100644
--- a/index.go
+++ b/index.go
@@ -4,6 +4,8 @@ import (
 	"strings"
 )
 
+const HunkPathSeparator = "/"
+
 // BuildDirIndex mutates the Document to set the DirEnt field.
 //
 // The order of ChildrenList in the DirEnt
@@ -16,7 +18,7 @@ import (
 func (doc *Document) BuildDirIndex() {
 	doc.DirEnt = &DirEnt{}
 	for _, hunk := range doc.DataHunks {
-		doc.DirEnt.fill(strings.Split(hunk.Name, "/"), 0, hunk.Hunk)
+		doc.DirEnt.fill(strings.Split(hunk.Name, HunkPathSeparator), 0, hunk.Hunk)
 	}
 }
 
@@ -36,7 +38,7 @@ func (dirent *DirEnt) fill(pathSegs []string, pathIdx int, hunk Hunk) {
 	l := len(dirent.ChildrenList)
 	dirent.ChildrenList = append(dirent.ChildrenList, &DirEnt{
 		Name: pathSegs[pathIdx],
-		Path: strings.Join(pathSegs[:pathIdx+1], "/"),
+		Path: strings.Join(pathSegs[:pathIdx+1], HunkPathSeparator),
 	})
 	dirent.Children[pathSegs[pathIdx]] = dirent.ChildrenList[l]
 	dirent.ChildrenList[l].fill(pathSegs, pathIdx+1, hunk)
diff --git a/read_test.go b/read_test.go
index 41d1c12..62de99f 100644
--- a/read_test.go
+++ b/read_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	"github.com/warpfork/go-testmark"
@@ -54,3 +55,23 @@ func TestParseCRLF(t *testing.T) {
 
 	readFixturesExample(t, doc)
 }
+
+func TestDuplicateHunk(t *testing.T) {
+	testdata, err := filepath.Abs("testdata")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mdPath := filepath.Join(testdata, "exampleWithDuplicateHunks.md")
+	_, err = testmark.ReadFile(mdPath)
+	assertRegex(t, err.Error(), "repeated testmark hunk name \"one/two/three\".*")
+}
+
+func assertRegex(t testing.TB, actual string, pattern string) {
+	matched, err := regexp.MatchString(pattern, actual)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !matched {
+		t.Errorf("expected %q to match pattern %q", actual, pattern)
+	}
+}
diff --git a/testdata/exampleWithDuplicateHunks.md b/testdata/exampleWithDuplicateHunks.md
new file mode 100644
index 0000000..c2b5cab
--- /dev/null
+++ b/testdata/exampleWithDuplicateHunks.md
@@ -0,0 +1,15 @@
+an example with duplicate hunk names
+====
+
+Duplicate hunk names are invalid and will cause parse to return an error.
+
+  
+[testmark]:# (one/two/three)
+```text
+foo
+```
+
+[testmark]:# (one/two/three)
+```text
+bar
+```