language composition: classification, aggregation, bounded cache
commit 370d8f1c9f71a485e1d3fab6f6edde90576dc8a2 · parent 2bf3fecf7c
· browse this tree
Ximianth, 2026-10-26 23:00:13 UTC
Composition by bytes at a commit, classified from the recursive tree listing by go-enry (the Go linguist port; Apache-2.0, pure Go on its default build path, no new transitive modules). Filename-first: only paths whose name leaves several candidate languages get a capped content read, batched through one cat-file --batch process. Vendored, dotfile, documentation, and generated paths are skipped per linguist heuristics; only programming and markup languages count, so manifests and prose don't swamp the result. Cached per (repo, sha), bounded, FIFO: derived data, free to lose.
diff --git a/git.go b/git.go index dd03435..a7ad223 100644 --- a/git.go +++ b/git.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "context" "errors" @@ -224,6 +225,23 @@ func (r *repo) lsTree(sha, path string) ([]treeEntry, error) { if err != nil { return nil, notFoundOnExit(err, fmt.Sprintf("directory %q at %s", path, sha)) } + return parseLsTree(out) +} + +// lsTreeRecursive lists every entry of the commit's whole tree, depth first; +// each entry's name carries its full path from the root. Directories themselves +// are omitted (git's -r behavior), so the result is blobs and submodule pointers. +// sha is a full commit sha as returned by resolveCommit. +func (r *repo) lsTreeRecursive(sha string) ([]treeEntry, error) { + out, err := r.git("ls-tree", "-z", "-l", "-r", sha+":") + if err != nil { + return nil, notFoundOnExit(err, fmt.Sprintf("tree at %s", sha)) + } + return parseLsTree(out) +} + +// parseLsTree parses NUL-terminated `ls-tree -z -l` output. +func parseLsTree(out []byte) ([]treeEntry, error) { var entries []treeEntry for _, rec := range strings.Split(string(out), "\x00") { if rec == "" { @@ -237,6 +255,7 @@ func (r *repo) lsTree(sha, path string) ([]treeEntry, error) { } size := int64(-1) if fields[3] != "-" { + var err error size, err = strconv.ParseInt(fields[3], 10, 64) if err != nil { return nil, fmt.Errorf("unexpected ls-tree size in record %q", rec) @@ -280,6 +299,69 @@ func (r *repo) catBlob(sha, path string, w io.Writer) error { return nil } +// catBlobsCapped reads several blobs through one cat-file process, handing the first +// capBytes of each to visit, in the order requested. The full object still crosses the +// pipe (--batch has no partial reads), so the cap bounds memory, not I/O. +// hashes are expected to come straight out of ls-tree on the same repo, so a missing or +// non-blob object is an error, not a soft miss. +func (r *repo) catBlobsCapped(hashes []string, capBytes int64, visit func(hash string, head []byte)) error { + argv := []string{"--git-dir=" + r.gitDir, "cat-file", "--batch"} + cmd := gitCommand(context.Background(), argv...) + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + return err + } + // Reap the subprocess on every path; on early error returns the kill also + // unblocks the feeding goroutine's pipe writes. + defer func() { + cmd.Process.Kill() + cmd.Wait() + }() + go func() { + for _, h := range hashes { + if _, err := io.WriteString(stdin, h+"\n"); err != nil { + break + } + } + stdin.Close() + }() + br := bufio.NewReader(stdout) + for _, h := range hashes { + line, err := br.ReadString('\n') + if err != nil { + return newGitError(argv, fmt.Errorf("reading batch header for %s: %w", h, err), stderr.String()) + } + // A good header is "<hash> blob <size>"; anything else is a hard mismatch. + fields := strings.Fields(line) + if len(fields) != 3 || fields[0] != h || fields[1] != "blob" { + return newGitError(argv, fmt.Errorf("unexpected batch header %q for %s", strings.TrimSpace(line), h), stderr.String()) + } + size, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return newGitError(argv, fmt.Errorf("unexpected batch size in %q", strings.TrimSpace(line)), stderr.String()) + } + head := make([]byte, min(size, capBytes)) + if _, err := io.ReadFull(br, head); err != nil { + return newGitError(argv, fmt.Errorf("reading blob %s: %w", h, err), stderr.String()) + } + // Drain the remainder and the record's trailing newline. + if _, err := io.CopyN(io.Discard, br, size-int64(len(head))+1); err != nil { + return newGitError(argv, fmt.Errorf("draining blob %s: %w", h, err), stderr.String()) + } + visit(h, head) + } + return nil +} + // commitInfo is one entry of a commit log. type commitInfo struct { hash string diff --git a/go.mod b/go.mod index 0f45e20..93b355c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module socle go 1.26.5 require ( + github.com/go-enry/go-enry/v2 v2.9.6 github.com/warpfork/go-testmark v0.12.1 gopkg.in/yaml.v3 v3.0.1 vitrine v0.0.0 @@ -12,6 +13,7 @@ require ( github.com/disintegration/imaging v1.6.2 // indirect github.com/flopp/go-findfont v0.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/goccy/go-graphviz v0.2.10 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/tetratelabs/wazero v1.10.1 // indirect diff --git a/go.sum b/go.sum index 4dbf992..2d61ac3 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,33 @@ github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI= github.com/corona10/goimagehash v1.1.0/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/flopp/go-findfont v0.1.0 h1:lPn0BymDUtJo+ZkV01VS3661HL6F4qFlkhcJN55u6mU= github.com/flopp/go-findfont v0.1.0/go.mod h1:wKKxRDjD024Rh7VMwoU90i6ikQRCr+JTHB5n4Ejkqvw= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/go-enry/go-enry/v2 v2.9.6 h1:np63eOtMV56zfYDHnFVgpEVOk8fr2kmylcMnAZUDbSs= +github.com/go-enry/go-enry/v2 v2.9.6/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/goccy/go-graphviz v0.2.10 h1:jHu/1I0Iw0xIzzYk96Ous/ZeuD11Rt2oW8juHdIE30g= github.com/goccy/go-graphviz v0.2.10/go.mod h1:LRlMnNmY17QbN6fLnvOzY7g0rXQjLKAhzxeTHbEUM6w= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= github.com/warpfork/go-fsx v0.3.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg= @@ -27,5 +43,7 @@ golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lang.go b/lang.go new file mode 100644 index 0000000..f9de2d7 --- /dev/null +++ b/lang.go @@ -0,0 +1,178 @@ +package main + +import ( + "sort" + "sync" + + "github.com/go-enry/go-enry/v2" +) + +// This file computes a repo's language composition at a commit: the recursive tree +// listing classified by enry (the Go linguist port), aggregated by blob bytes. +// Classification is filename-first; only paths whose filename leaves several candidate +// languages get a capped content read, batched through one cat-file process. +// Results are cached per (repo, sha) in bounded memory: derived data, free to lose. + +// sniffCap is how many leading bytes of a blob the classifier sees when the filename +// alone is ambiguous. Linguist reads 51200 for the same purpose; a smaller head is +// plenty for telling candidate languages apart, and bounds the memory a sniff holds. +const sniffCap = 16 * 1024 + +// langCacheCap bounds the composition cache. Entries are a handful of name+count pairs, +// so this is a few KiB in total; the cap exists so shas accumulated over a long uptime +// (every pushed commit a visitor looks at is a fresh key) cannot grow without limit. +const langCacheCap = 256 + +// langStat is one language's byte count at a commit. +type langStat struct { + name string + bytes int64 +} + +// classifyPath names the language of one path by its name alone. +// needsContent reports that several candidate languages remain: the caller should +// re-classify with a capped read of the content (and a lang of "" alongside). +// A plain empty lang means the path contributes nothing: unclassifiable, or skipped +// as vendored, dotfile, documentation, or generated, per enry's linguist heuristics. +func classifyPath(path string) (lang string, needsContent bool) { + if enry.IsVendor(path) || enry.IsDotFile(path) || enry.IsDocumentation(path) || enry.IsGenerated(path, nil) { + return "", false + } + langs := enry.GetLanguages(path, nil) + switch len(langs) { + case 0: + return "", false + case 1: + return langs[0], false + default: + return "", true + } +} + +// countsToward reports whether a language belongs in the composition meter: +// programming and markup languages do; data and prose do not (linguist's convention, +// so manifests, configs, and READMEs don't swamp the bar). +func countsToward(lang string) bool { + switch enry.GetLanguageType(lang) { + case enry.Programming, enry.Markup: + return true + } + return false +} + +// languageStats computes the language composition at a commit, by bytes, +// sorted by descending byte count (ties by name, for determinism). +// sha is a full commit sha as returned by resolveCommit. +func languageStats(r *repo, sha string) ([]langStat, error) { + entries, err := r.lsTreeRecursive(sha) + if err != nil { + return nil, err + } + totals := map[string]int64{} + count := func(lang string, bytes int64) { + if countsToward(lang) { + totals[lang] += bytes + } + } + // Pass one: names alone. Ambiguous paths queue for the content pass, + // grouped by hash so identical blobs cross the pipe once. + pathsByHash := map[string][]string{} + var sniffHashes []string + for _, e := range entries { + // Only regular blobs hold language bytes: symlink blobs are just their + // target path, and submodule pointers have no content here at all. + if e.typ != "blob" || e.mode == "120000" { + continue + } + lang, needsContent := classifyPath(e.name) + switch { + case needsContent: + if _, seen := pathsByHash[e.hash]; !seen { + sniffHashes = append(sniffHashes, e.hash) + } + pathsByHash[e.hash] = append(pathsByHash[e.hash], e.name) + case lang != "": + count(lang, e.size) + } + } + // Pass two: capped content for the ambiguous ones. + if len(sniffHashes) > 0 { + sizes := map[string]int64{} + for _, e := range entries { + sizes[e.hash] = e.size + } + err := r.catBlobsCapped(sniffHashes, sniffCap, func(hash string, head []byte) { + for _, path := range pathsByHash[hash] { + if lang := enry.GetLanguage(path, head); lang != "" { + count(lang, sizes[hash]) + } + } + }) + if err != nil { + return nil, err + } + } + stats := make([]langStat, 0, len(totals)) + for name, bytes := range totals { + stats = append(stats, langStat{name, bytes}) + } + sort.Slice(stats, func(i, j int) bool { + if stats[i].bytes != stats[j].bytes { + return stats[i].bytes > stats[j].bytes + } + return stats[i].name < stats[j].name + }) + return stats, nil +} + +// langCache holds computed compositions, bounded, with FIFO eviction. +// FIFO rather than LRU because the hit pattern is dominated by one key per repo +// (the current HEAD); recency tracking would buy nothing for its bookkeeping. +type langCache struct { + mu sync.Mutex + cap int + order []string // insertion order; order[0] is the next to evict + m map[string][]langStat +} + +func newLangCache(capacity int) *langCache { + return &langCache{cap: capacity, m: make(map[string][]langStat, capacity)} +} + +func langCacheKey(name, sha string) string { return name + "\x00" + sha } + +func (c *langCache) get(key string) ([]langStat, bool) { + c.mu.Lock() + defer c.mu.Unlock() + stats, ok := c.m[key] + return stats, ok +} + +func (c *langCache) put(key string, stats []langStat) { + c.mu.Lock() + defer c.mu.Unlock() + if _, exists := c.m[key]; !exists { + if len(c.order) >= c.cap { + delete(c.m, c.order[0]) + c.order = c.order[1:] + } + c.order = append(c.order, key) + } + c.m[key] = stats +} + +// langStats is the serving layer's entry point: languageStats through the cache. +// Concurrent misses on the same key may compute twice; the computation is idempotent +// and the duplicate work is cheaper than single-flight bookkeeping. +func (s *server) langStats(rp *repo, sha string) ([]langStat, error) { + key := langCacheKey(rp.name, sha) + if stats, ok := s.langs.get(key); ok { + return stats, nil + } + stats, err := languageStats(rp, sha) + if err != nil { + return nil, err + } + s.langs.put(key, stats) + return stats, nil +} diff --git a/lang_test.go b/lang_test.go new file mode 100644 index 0000000..7d2b947 --- /dev/null +++ b/lang_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "testing" +) + +// TestClassifyPath pins the filename-only classification pass, one row per rule: +// plain classification, the markup case, the skip heuristics (vendored, dotfile, +// documentation), unclassifiable paths, and the ambiguous-extension escape hatch. +func TestClassifyPath(t *testing.T) { + t.Parallel() + for _, c := range []struct { + path string + lang string + needsContent bool + }{ + {"src/hello.go", "Go", false}, + {"src/style.css", "CSS", false}, + {"Makefile", "Makefile", false}, + {"design.md", "Markdown", false}, // classified fine; prose is excluded later, by countsToward + {"src/lib.rs", "", true}, // several candidate languages: classify by content + {"vendor/mini.js", "", false}, // vendored + {"testdata/gen.go", "", false}, // vendored (linguist treats test fixtures so) + {".gitignore", "", false}, // dotfile + {"docs/guide.html", "", false}, // documentation + {"README.md", "", false}, // documentation too, READMEs count as docs + {"LICENSE", "", false}, // unclassifiable by name + {"notes", "", false}, // no extension, no known filename + } { + lang, needsContent := classifyPath(c.path) + if lang != c.lang || needsContent != c.needsContent { + t.Errorf("classifyPath(%q) = %q, %v; want %q, %v", c.path, lang, needsContent, c.lang, c.needsContent) + } + } +} + +// TestCountsToward pins the type filter: programming and markup make the meter; +// data, prose, and the unknown empty string do not. +func TestCountsToward(t *testing.T) { + t.Parallel() + for _, c := range []struct { + lang string + want bool + }{ + {"Go", true}, + {"Rust", true}, + {"CSS", true}, // markup + {"HTML", true}, // markup + {"Markdown", false}, // prose + {"Text", false}, // prose + {"YAML", false}, // data + {"Go Module", false}, // data + {"", false}, + } { + if got := countsToward(c.lang); got != c.want { + t.Errorf("countsToward(%q) = %v, want %v", c.lang, got, c.want) + } + } +} + +// TestLanguageStats runs the whole computation against the fixture repo: +// Go, Rust (via the content sniff), and CSS counted by blob size, descending; +// prose, data, vendored, and dotfile content absent. +func TestLanguageStats(t *testing.T) { + t.Parallel() + r := fixtureRepo(t) + size := func(path string) int64 { + t.Helper() + n, err := r.blobSize(fixtureShaMain, path) + if err != nil { + t.Fatal(err) + } + return n + } + want := []langStat{ + {"Go", size("src/hello.go")}, + {"Rust", size("src/lib.rs")}, + {"CSS", size("src/style.css")}, + } + got, err := languageStats(r, fixtureShaMain) + if err != nil { + t.Fatal(err) + } + if len(got) != len(want) { + t.Fatalf("languageStats = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("stat %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +// TestLangCacheBounds pins the cache's FIFO eviction: the capacity holds, +// the oldest key leaves first, and lookups tell hits from misses. +func TestLangCacheBounds(t *testing.T) { + t.Parallel() + c := newLangCache(2) + c.put("a", []langStat{{"Go", 1}}) + c.put("b", []langStat{{"Go", 2}}) + if _, ok := c.get("a"); !ok { + t.Error("a missing before capacity was exceeded") + } + c.put("c", []langStat{{"Go", 3}}) + if _, ok := c.get("a"); ok { + t.Error("a survived eviction; want FIFO drop of the oldest") + } + for _, key := range []string{"b", "c"} { + if _, ok := c.get(key); !ok { + t.Errorf("%s missing after eviction of a", key) + } + } + if _, ok := c.get("never"); ok { + t.Error("get invented an entry") + } +} + +// TestLangStatsCached proves the server-level path consults the cache: +// a planted cache entry is returned instead of a fresh computation. +func TestLangStatsCached(t *testing.T) { + t.Parallel() + s := fixtureServer(t) + rp := s.repos["fixturama"] + planted := []langStat{{"Ada", 42}} + s.langs.put(langCacheKey(rp.name, fixtureShaMain), planted) + got, err := s.langStats(rp, fixtureShaMain) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != planted[0] { + t.Errorf("langStats = %+v, want the planted %+v", got, planted) + } +} diff --git a/serve.go b/serve.go index 58bc338..ec07733 100644 --- a/serve.go +++ b/serve.go @@ -19,6 +19,7 @@ type server struct { theme display.Theme // fixed for the process lifetime; see display/theme.go sketch map[string][]byte // the generated sketch SVGs, by filename; see sketch.go style []byte // the assembled /-/style.css body; see buildStyle + langs *langCache // language composition per (repo, sha); see lang.go } func newServer(repos map[string]*repo, th display.Theme) *server { @@ -27,6 +28,7 @@ func newServer(repos map[string]*repo, th display.Theme) *server { theme: th, sketch: sketchAssets(th.Name), style: buildStyle(th), + langs: newLangCache(langCacheCap), } }