testmark-go

I may or may not have started parsing markdown, badly, for nefarious purposes.

commit 139314a9b3fe841c273767711a0c459032870069 · browse this tree
Eric Myhre, 2021-08-17 17:29:16 UTC

diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..dc56ee0
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,3 @@
+module github.com/warpfork/go-testmark
+
+go 1.16
diff --git a/read.go b/read.go
new file mode 100644
index 0000000..086e624
--- /dev/null
+++ b/read.go
@@ -0,0 +1,93 @@
+package testmark
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"unicode"
+)
+
+func Read(r io.Reader) (*Document, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	return Parse(data)
+}
+
+func ReadFile(name string) (*Document, error) {
+	data, err := os.ReadFile(name)
+	if err != nil {
+		return nil, err
+	}
+	return Parse(data)
+}
+
+var (
+	sigilLineBreak = []byte{'\n'}
+	sigilCodeBlock = []byte("```")
+	sigilTestmark  = []byte("[testmark]:# ")
+)
+
+func Parse(data []byte) (*Document, error) {
+	doc := Document{Original: data}
+
+	// Markdown can be effectively parsed line by line.
+	doc.OriginalLines = bytes.Split(data, sigilLineBreak)
+
+	// The first layer of our parse is "is this in a code block".
+	// Code blocks are the only feature of markdown that meaningfully changes what mode you're in at the start of a line.
+	// After that, we look for our magic prefix (on any lines that *aren't* already in a code block).
+	// Then, the rest is... pretty straightforward.
+	var inCodeBlock bool
+	var expectCodeBlock bool
+	hunkInProgress := Hunk{Line: -1}
+	for i, line := range doc.OriginalLines {
+		// Check for transition in or out of codeblock.
+		if bytes.HasPrefix(line, sigilCodeBlock) {
+			switch inCodeBlock {
+			case false: // starting a block
+				if expectCodeBlock {
+					hunkInProgress.BlockTag = string(line[len(sigilCodeBlock):])
+				}
+				expectCodeBlock = false
+			case true: // ending a block
+				if hunkInProgress.Line > -1 {
+					hunkInProgress.EndLine = i
+					doc.DataHunks = append(doc.DataHunks, hunkInProgress)
+					hunkInProgress = Hunk{Line: -1}
+				}
+			}
+			inCodeBlock = !inCodeBlock
+			continue
+		}
+		// If we're in a code block, just fly by.
+		if inCodeBlock {
+			continue
+		}
+		// If we were expecting a code block just now, we didn't get it.
+		if expectCodeBlock {
+			// ... Log a complaint?  I don't think halting with a parse error would be helpful.
+			// But definitely don't wait around for arbitrarily distant code blocks.
+			expectCodeBlock = false
+		}
+		// Look for testmark block indicators.
+		if bytes.HasPrefix(line, sigilTestmark) {
+			remainder := line[len(sigilTestmark):]
+			nameEnd := bytes.IndexFunc(remainder, unicode.IsSpace)
+			if nameEnd < 0 {
+				nameEnd = len(remainder)
+			}
+			name := remainder[0:nameEnd]
+			if len(name) == 0 {
+				// ... Log a complaint?  Empty block names are very silly.
+			}
+			expectCodeBlock = true
+			hunkInProgress.Line = i
+			hunkInProgress.Name = string(name)
+		}
+		// Any other text?  It's prose.  No action.
+	}
+	return &doc, nil
+}
diff --git a/read_test.go b/read_test.go
new file mode 100644
index 0000000..5e8a7c0
--- /dev/null
+++ b/read_test.go
@@ -0,0 +1,16 @@
+package testmark
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+)
+
+func TestRead(t *testing.T) {
+	testdata, err := filepath.Abs("testdata")
+	if err != nil {
+		panic(err)
+	}
+	doc, err := ReadFile(filepath.Join(testdata, "example.md"))
+	fmt.Printf("%v -- %v\n", doc.DataHunks, err)
+}
diff --git a/testdata/example.md b/testdata/example.md
new file mode 100644
index 0000000..f2742b0
--- /dev/null
+++ b/testdata/example.md
@@ -0,0 +1,73 @@
+this is a markdown file
+=======================
+
+... and it's also going to contain test fixture data.
+
+The markdown is full of idioms and unidiomatic things alike.
+
+The contents don't really matter, mostly.
+They're freetext, and you can use markdown to describe whatever you want.
+
+Except these:
+
+[testmark]:# this-is-the-data-name
+```text
+the content of this code block is data which can be read,
+and *replaced*, by testmark.
+```
+
+That's not a regular code block.
+I mean, it is -- but be sure to look at this file in a "raw" mode.
+There's also a comment above it, which tells testmark to look at it:
+
+```
+[testmark]:# this-is-the-data-name
+```
+
+That comment, coming right before a triple-backtick code block, is a signal.
+It tells textmark to look at the codeblock, and also gives that code block a name.
+
+
+Multiple data hunks per file
+----------------------------
+
+You can have more than one block like that in a file:
+
+[testmark]:# more-data
+```go
+func OtherMarkdownParsers() (shouldHighlight bool) {
+	return true
+}
+```
+
+### the headings don't matter
+
+Every other markdown feature, like the headings, are totally ignored.
+That structure is for *you*, human, as you write documentation together with your data.
+
+
+Editing
+-------
+
+Testmark can edit a file like this one, and replace the code block contents according to the name given in the comment.
+(It's kind of like a big map of strings, in that regard.)
+
+Usually, a human writes the testmark file.
+If the human wants to programmatically populate things, the human writes out the code block and names it with the magic comment format,
+and then runs some tool that updates the content.
+
+Some libraries may also be able to create a testmark file purely programmatically, but this is usually more complicated,
+and makes it harder to control the rest of the document...
+which, presumably, you do still want to fill with prose (and markdown-formatted) descriptions of the data.
+
+
+One note
+--------
+
+There is one thing this format is bad at:
+you can't easily describe data that doesn't have a trailing linebreak.
+
+[testmark]:# cannot-describe-no-linebreak
+```
+A markdown codeblock always has a trailing linebreak before its close indicator, you see.
+```
diff --git a/testmark.go b/testmark.go
new file mode 100644
index 0000000..2cefe63
--- /dev/null
+++ b/testmark.go
@@ -0,0 +1,38 @@
+package testmark
+
+type Document struct {
+	// The whole thing, complete, but split into lines.
+	// We always save this, because if we are going to write this document back out,
+	// it's going to be by patching this.  (We don't try to understand, much less normalize, a full markdown AST!)
+	Original []byte
+
+	// Original, sliced into lines.  Shares backing memory with Original.
+	// Useful because we made it during parse anyway, and it can save us a lot of work during edits.
+	OriginalLines [][]byte
+
+	// Each data hunk.
+	// Contains just offset information, and the parsed name header.
+	DataHunks []Hunk
+
+	// Like it says on the tin.
+	HunksByName map[string]Hunk
+}
+
+type Hunk struct {
+	// Index into Document.OriginalLines where the comment block is found.
+	// The code block indicator is necessarily is the following line,
+	// and the code block body one line after that.
+	// N.B. zero-indexed.  You probably want to +1 before printing to a human.
+	Line int
+
+	// Index into Document.OriginalLines that contains the closing code block indicator.
+	EndLine int
+
+	// The hunk name (e.g. whatever comes after `[testmark]:# ` and before any more whitespace).
+	// Cannot be empty.
+	Name string
+
+	// The code block syntax hint (or more literally: anything that comes after the triple-tick that starts the code block).
+	// Usually we don't encourage use of this much in testmark, but it's here.  Can be empty.
+	BlockTag string
+}