lang.go

package main

import (
	"sort"
	"sync"

	"github.com/go-enry/go-enry/v2"
)

// This file computes a repo's language composition at a commit: the recursive tree
// listing classified by enry (the Go linguist port), aggregated by blob bytes.
// Classification is filename-first; only paths whose filename leaves several candidate
// languages get a capped content read, batched through one cat-file process.
// Results are cached per (repo, sha) in bounded memory: derived data, free to lose.

// sniffCap is how many leading bytes of a blob the classifier sees when the filename
// alone is ambiguous.  Linguist reads 51200 for the same purpose; a smaller head is
// plenty for telling candidate languages apart, and bounds the memory a sniff holds.
const sniffCap = 16 * 1024

// langCacheCap bounds the composition cache.  Entries are a handful of name+count pairs,
// so this is a few KiB in total; the cap exists so shas accumulated over a long uptime
// (every pushed commit a visitor looks at is a fresh key) cannot grow without limit.
const langCacheCap = 256

// langStat is one language's byte count at a commit.
type langStat struct {
	name  string
	bytes int64
}

// classifyPath names the language of one path by its name alone.
// needsContent reports that several candidate languages remain: the caller should
// re-classify with a capped read of the content (and a lang of "" alongside).
// A plain empty lang means the path contributes nothing: unclassifiable, or skipped
// as vendored, dotfile, documentation, or generated, per enry's linguist heuristics.
func classifyPath(path string) (lang string, needsContent bool) {
	if enry.IsVendor(path) || enry.IsDotFile(path) || enry.IsDocumentation(path) || enry.IsGenerated(path, nil) {
		return "", false
	}
	langs := enry.GetLanguages(path, nil)
	switch len(langs) {
	case 0:
		return "", false
	case 1:
		return langs[0], false
	default:
		return "", true
	}
}

// countsToward reports whether a language belongs in the composition meter:
// programming and markup languages do; data and prose do not (linguist's convention,
// so manifests, configs, and READMEs don't swamp the bar).
func countsToward(lang string) bool {
	switch enry.GetLanguageType(lang) {
	case enry.Programming, enry.Markup:
		return true
	}
	return false
}

// languageStats computes the language composition at a commit, by bytes,
// sorted by descending byte count (ties by name, for determinism).
// sha is a full commit sha as returned by resolveCommit.
func languageStats(r *repo, sha string) ([]langStat, error) {
	entries, err := r.lsTreeRecursive(sha)
	if err != nil {
		return nil, err
	}
	totals := map[string]int64{}
	count := func(lang string, bytes int64) {
		if countsToward(lang) {
			totals[lang] += bytes
		}
	}
	// Pass one: names alone.  Ambiguous paths queue for the content pass,
	// grouped by hash so identical blobs cross the pipe once.
	pathsByHash := map[string][]string{}
	var sniffHashes []string
	for _, e := range entries {
		// Only regular blobs hold language bytes: symlink blobs are just their
		// target path, and submodule pointers have no content here at all.
		if e.typ != "blob" || e.mode == "120000" {
			continue
		}
		lang, needsContent := classifyPath(e.name)
		switch {
		case needsContent:
			if _, seen := pathsByHash[e.hash]; !seen {
				sniffHashes = append(sniffHashes, e.hash)
			}
			pathsByHash[e.hash] = append(pathsByHash[e.hash], e.name)
		case lang != "":
			count(lang, e.size)
		}
	}
	// Pass two: capped content for the ambiguous ones.
	if len(sniffHashes) > 0 {
		sizes := map[string]int64{}
		for _, e := range entries {
			sizes[e.hash] = e.size
		}
		err := r.catBlobsCapped(sniffHashes, sniffCap, func(hash string, head []byte) {
			for _, path := range pathsByHash[hash] {
				if lang := enry.GetLanguage(path, head); lang != "" {
					count(lang, sizes[hash])
				}
			}
		})
		if err != nil {
			return nil, err
		}
	}
	stats := make([]langStat, 0, len(totals))
	for name, bytes := range totals {
		stats = append(stats, langStat{name, bytes})
	}
	sort.Slice(stats, func(i, j int) bool {
		if stats[i].bytes != stats[j].bytes {
			return stats[i].bytes > stats[j].bytes
		}
		return stats[i].name < stats[j].name
	})
	return stats, nil
}

// langCache holds computed compositions, bounded, with FIFO eviction.
// FIFO rather than LRU because the hit pattern is dominated by one key per repo
// (the current HEAD); recency tracking would buy nothing for its bookkeeping.
type langCache struct {
	mu    sync.Mutex
	cap   int
	order []string // insertion order; order[0] is the next to evict
	m     map[string][]langStat
}

func newLangCache(capacity int) *langCache {
	return &langCache{cap: capacity, m: make(map[string][]langStat, capacity)}
}

func langCacheKey(name, sha string) string { return name + "\x00" + sha }

func (c *langCache) get(key string) ([]langStat, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	stats, ok := c.m[key]
	return stats, ok
}

func (c *langCache) put(key string, stats []langStat) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if _, exists := c.m[key]; !exists {
		if len(c.order) >= c.cap {
			delete(c.m, c.order[0])
			c.order = c.order[1:]
		}
		c.order = append(c.order, key)
	}
	c.m[key] = stats
}

// langStats is the serving layer's entry point: languageStats through the cache.
// Concurrent misses on the same key may compute twice; the computation is idempotent
// and the duplicate work is cheaper than single-flight bookkeeping.
func (s *server) langStats(rp *repo, sha string) ([]langStat, error) {
	key := langCacheKey(rp.name, sha)
	if stats, ok := s.langs.get(key); ok {
		return stats, nil
	}
	stats, err := languageStats(rp, sha)
	if err != nil {
		return nil, err
	}
	s.langs.put(key, stats)
	return stats, nil
}