orient-files

#!/bin/bash
# Classifies files by type and counts them across a project area.
#
# Usage: orient-files [-x|--list-unclassified] [PATH]
#   PATH defaults to the git toplevel of cwd, else cwd itself.
#
# Output: a bullet list, one row per type, sorted by count descending.
# Each row has three columns: type name, count, and a hint indicating
# which heuristic(s) matched.  Types with fewer than 3 files are lumped
# into a trailing "(other)" line so the output stays bounded.
#
# Example output:
#   - markdown:         25  (`.md`)
#   - typescript:     1241  (`.ts`)
#   - bash:             14  (shebang, `.bash`)
#   - image:           108  (`.png`, `.jpg`, `.gif`)
#   - unknown-shell:     7  (`.sh`)
#   - (other): 9 files across 5 kinds
#
# Type names are canonical (lowercased) language/format names, chosen
# for legibility when scanning.  Extensions without a mapping fall
# through to using the literal extension as the type name, so unfamiliar
# projects stay informative without needing the table to be exhaustive.
#
# Heuristics:
# - Extension: a small handwritten table mapping `.md` -> markdown etc.
# - Shebang: for executable regular files, the first 100 bytes are
#   read and the first line is matched against a known interpreter list
#   (bash, sh, zsh, dash, ksh, python, node, perl, ruby, lua, awk).
#   Shebang dominates extension when both produce a verdict: a `.sh`
#   file with `#!/usr/bin/env python3` counts as python.
#   Only executable files get a shebang sniff, which keeps the cost
#   bounded -- almost all source files are non-executable.
# The hint column lists which heuristics fired for a given type: it's
# a tally of "reasons we looked", not a derivation trace.
#
# Extensionless files (e.g. `LICENSE`, `Makefile`) and dotfiles whose
# name carries no further dot (`.gitignore`, `.editorconfig`) are folded
# into `(other)` along with files of rare extensions.
# With `-x`/`--list-unclassified`, every file that landed in `(other)`
# is itemized as nested bullets, grouped by type, so it's clear what
# the lump actually contains.  In practice this is a useful peek: the
# `(other)` bucket often hides dotfile-shaped config that's central to
# a project's identity.
#
# Silent (no output, exit 0) when no countable files are found.
#
# Filtering:
# - Dotdirs (`.git`, `.venv`, etc.) are pruned entirely.
# - A small list of well-known build/vendor dirs is also pruned
#   (`node_modules`, `target`, `dist`, `build`, `vendor`).
#   The bias is conservative: rather miss a real file under a misnamed
#   `vendor/` than have `node_modules` dominate every count.

set -euo pipefail

list_unclassified=0
while [ $# -gt 0 ]; do
	case "$1" in
		-x|--list-unclassified) list_unclassified=1; shift ;;
		-h|--help)
			sed -n '2,/^$/p' "$0" | sed 's/^# \?//'
			exit 0 ;;
		--) shift; break ;;
		-*) echo "orient-files: unknown option: $1" >&2; exit 2 ;;
		*) break ;;
	esac
done

# Resolve the search root via the single source of truth.
# `find-project-root` is strict (git toplevel or exit 1); we add a cwd
# fallback here since classifying files outside a project is still useful.
self_dir=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
if [ $# -ge 1 ]; then
	root=$(cd "$1" && pwd)
elif root=$("$self_dir/find-project-root" 2>/dev/null); then
	:
else
	root="$PWD"
fi

# Extension -> canonical type name.
# Names roughly follow github-linguist's language list (consulted for
# inspiration, not pulled in as a dependency).
# `.sh` maps to `unknown-shell` rather than `bash`: the actual interpreter
# of a bare `.sh` file depends on the system `/bin/sh`, which varies by
# distro, so flagging that ambiguity is more honest than guessing.
# Shebang-based detection (in a later commit) will resolve specific shells.
declare -A ext_to_type=(
	[.md]=markdown        [.markdown]=markdown
	[.rs]=rust
	[.py]=python
	[.ts]=typescript      [.tsx]=typescript
	[.js]=javascript      [.mjs]=javascript    [.cjs]=javascript    [.jsx]=javascript
	[.go]=go
	[.v]=vlang
	[.c]=c                [.h]=c
	[.cpp]=cpp            [.cc]=cpp            [.cxx]=cpp           [.hpp]=cpp           [.hh]=cpp
	[.java]=java
	[.rb]=ruby
	[.php]=php
	[.cs]=csharp
	[.swift]=swift
	[.kt]=kotlin          [.kts]=kotlin
	[.lua]=lua
	[.zig]=zig
	[.dart]=dart
	[.scala]=scala
	[.ex]=elixir          [.exs]=elixir
	[.erl]=erlang
	[.hs]=haskell
	[.clj]=clojure        [.cljs]=clojure
	[.r]=r                [.R]=r
	[.jl]=julia
	[.pl]=perl            [.pm]=perl
	[.sql]=sql
	[.sh]=unknown-shell   [.bash]=bash
	[.nu]=nushell
	[.json]=json
	[.yaml]=yaml          [.yml]=yaml
	[.toml]=toml
	[.html]=html          [.htm]=html
	[.css]=css
	[.xml]=xml
	[.ini]=ini
	[.csv]=csv            [.tsv]=tsv
	[.png]=image          [.jpg]=image         [.jpeg]=image        [.gif]=image
	[.svg]=image          [.webp]=image
	[.txt]=text
	[.log]=log
)

# Interpreter name (as it appears in a shebang) -> canonical type name.
# Order in the regex matters only for completeness, not for precedence:
# any single file's shebang names one interpreter.
declare -A interp_to_type=(
	[bash]=bash
	[sh]=sh
	[zsh]=zsh
	[dash]=dash
	[ksh]=ksh
	[python]=python   [python2]=python   [python3]=python
	[node]=javascript [nodejs]=javascript
	[perl]=perl
	[ruby]=ruby
	[lua]=lua
	[awk]=awk         [gawk]=awk         [mawk]=awk
)

# `find` permission-denied messages on unrelated subdirs are expected
# noise in shared filesystems (e.g. running under /tmp).  Filter those
# out, but pass any other stderr through unchanged so real problems
# still surface.
#
# Each found file is emitted as a TAB-separated triple:
#     <ext_or_(none)>\t<basename>\t<x_or_->
# where the trailing flag marks executable regular files (which then get
# a shebang sniff in a second pass).  We need both filename and path-ish
# info to be carried through, so we use -printf with %p (full path) for
# the shebang lookup; the basename comes from %f.
mapfile -t classified < <(
	find "$root" \
		\( \
			-type d \
			\( -name '.*' \
			   -o -name 'node_modules' \
			   -o -name 'target' \
			   -o -name 'dist' \
			   -o -name 'build' \
			   -o -name 'vendor' \
			\) -prune \
		\) -o \
		\( -type f \( -executable -printf 'x\t%f\t%p\n' \) \) -o \
		\( -type f -printf '-\t%f\t\n' \) \
		2> >(grep -v 'Permission denied' >&2) \
	| awk -F'\t' '
		function classify_ext(name,   classname, n, i, idx) {
			# For classification purposes, drop a leading dot so dotfiles
			# like `.gitignore` are treated as having no extension.
			classname = name
			if (substr(classname, 1, 1) == ".") classname = substr(classname, 2)
			n = length(classname)
			idx = 0
			for (i = n; i >= 1; i--) {
				if (substr(classname, i, 1) == ".") { idx = i; break }
			}
			if (idx == 0 || idx == n) return "(none)"
			return "." substr(classname, idx + 1)
		}
		{
			exec_flag = $1; name = $2; path = $3
			ext = classify_ext(name)
			# Output: <ext>\t<basename>\t<exec_flag>\t<path>
			# The basename is needed for the --list-unclassified listing.
			# The path is only used for the shebang sniff, and only when
			# exec_flag == "x"; otherwise it is empty.
			print ext "\t" name "\t" exec_flag "\t" path
		}
	'
)

# Silent if nothing to report.
if [ ${#classified[@]} -eq 0 ]; then
	exit 0
fi

# Sniff a file's shebang, cheaply.
# Read at most 100 bytes; require `#!` at byte 0; take the first line;
# match a known interpreter token via the basename of whatever follows
# the bang (handles both `#!/bin/bash` and `#!/usr/bin/env bash`).
# Echoes the canonical type name on a match, or nothing on a miss.
sniff_shebang() {
	local path=$1 head line interp
	# Strip NULs so bash doesn't complain about them in command substitution.
	# Binaries often start with a NUL run, and a real shebang line never
	# contains NULs anyway.
	head=$(head -c 100 -- "$path" 2>/dev/null | tr -d '\000') || return 0
	[[ "$head" == '#!'* ]] || return 0
	line=${head%%$'\n'*}
	# Match the rightmost basename-looking token in the shebang.
	# Examples it should pick up:
	#   #!/bin/bash              -> bash
	#   #!/usr/bin/env python3   -> python3
	#   #!/usr/bin/env -S bash   -> bash
	# Trailing flags (e.g. `python3 -u`) are stripped by taking the first
	# whitespace-delimited token after the last `/` or after `env`.
	# Strip leading `#!` and surrounding whitespace.
	line=${line#'#!'}
	line=${line## }
	# Split on whitespace; walk tokens to find the first that's a known
	# interpreter basename (skipping option words like `-S` and the
	# literal `env`).
	local tok base name
	for tok in $line; do
		# Skip option flags.
		[[ "$tok" == -* ]] && continue
		base=${tok##*/}
		# Skip the `env` dispatcher; the next token will be the interpreter.
		[[ "$base" == "env" ]] && continue
		name=${interp_to_type[$base]:-}
		if [ -n "$name" ]; then
			printf '%s' "$name"
			return 0
		fi
		# First non-flag, non-env token decided the interpreter (even if
		# we don't recognize it).  Bail rather than walking into argv.
		return 0
	done
}

# Partition into per-extension counts, the extensionless bucket, and
# shebang-driven reassignments.
#
# Per-type tallies are accumulated directly; for the hint column we
# track which heuristics fired and (for the extension heuristic) which
# extensions contributed.
# `type_counts[type]`      -- total files for that type.
# `type_exts[type]`        -- space-separated set of contributing extensions.
# `type_shebang[type]`     -- count of files whose shebang chose this type.
# `ext_counts[ext]`        -- per-extension count, used for sorting the hint
#                             list within a type (dominant variant first).
# `type_files[type]`       -- newline-separated filenames, populated only
#                             when --list-unclassified is set (the listing
#                             needs basenames per type; we skip the bookkeeping
#                             otherwise).
declare -A type_counts=()
declare -A type_exts=()
declare -A type_shebang=()
declare -A ext_counts=()
declare -A type_files=()
for entry in "${classified[@]}"; do
	IFS=$'\t' read -r ext name exec_flag path <<< "$entry"

	# Decide the type for this individual file.
	# Shebang dominates extension when both produce a verdict.
	shebang_type=""
	if [ "$exec_flag" = "x" ] && [ -n "$path" ]; then
		shebang_type=$(sniff_shebang "$path")
	fi

	if [ -n "$shebang_type" ]; then
		type=$shebang_type
		type_shebang[$type]=$((${type_shebang[$type]:-0} + 1))
		# Don't count this file's extension toward the type's ext set.
		# If the extension agreed with the shebang, we still want the row
		# to show that extension as a contributing heuristic -- but only
		# if some *other* file of the same extension landed here too.
		# That happens naturally below when non-shebang files of that
		# extension are processed.
	elif [ "$ext" = "(none)" ]; then
		# Extensionless files with no shebang verdict get their own type
		# bucket, so the downstream lumping and listing flow treats them
		# uniformly with other rare types.  Almost always small, almost
		# always interesting (dotfiles, LICENSE, Makefile, ...).
		type="extensionless"
	else
		type=${ext_to_type[$ext]:-$ext}
		ext_counts[$ext]=$((${ext_counts[$ext]:-0} + 1))
		if [ -z "${type_exts[$type]:-}" ]; then
			type_exts[$type]=$ext
		else
			# Add to set only if not already present.
			case " ${type_exts[$type]} " in
				*" $ext "*) ;;
				*) type_exts[$type]+=" $ext" ;;
			esac
		fi
	fi
	type_counts[$type]=$((${type_counts[$type]:-0} + 1))
	if [ "$list_unclassified" -eq 1 ]; then
		if [ -z "${type_files[$type]:-}" ]; then
			type_files[$type]=$name
		else
			type_files[$type]+=$'\n'$name
		fi
	fi
done

# Sort types by count descending, ties broken alphabetically.
mapfile -t sorted_types < <(
	for t in "${!type_counts[@]}"; do
		printf '%d\t%s\n' "${type_counts[$t]}" "$t"
	done | sort -k1,1rn -k2,2
)

# Split into shown vs. lumped buckets at the count >= 3 threshold.
# (Threshold chosen empirically: 1 or 2 files is rarely a meaningful
# signal about a project's character; 3 starts to suggest a pattern.)
shown_types=()
shown_counts=()
other_types=()
other_files=0
other_kinds=0
for line in "${sorted_types[@]}"; do
	count=${line%%$'\t'*}
	type=${line#*$'\t'}
	if [ "$count" -ge 3 ]; then
		shown_types+=("$type")
		shown_counts+=("$count")
	else
		other_types+=("$type")
		other_files=$((other_files + count))
		other_kinds=$((other_kinds + 1))
	fi
done

# Render the per-type rows.
#
# Column layout:
# - Type name (with trailing colon) left-padded to 16 chars.
#   Wide enough for `unknown-shell:` (14 chars); types longer than that
#   simply push the count column rightward on their own row.
# - Count right-aligned in 5 chars.
# - Two spaces, then the hint in parentheses.
#
# Hint composition: `shebang` appears first if the shebang heuristic
# fired for this type, then the contributing extensions sorted by
# per-extension count descending (dominant variant first), then
# alphabetically.  Determinism is important since bash assoc-array
# iteration order is unspecified.
for i in "${!shown_types[@]}"; do
	type=${shown_types[$i]}
	count=${shown_counts[$i]}
	hint_parts=()
	if [ -n "${type_shebang[$type]:-}" ]; then
		hint_parts+=("shebang")
	fi
	if [ -n "${type_exts[$type]:-}" ]; then
		mapfile -t type_ext_list < <(
			for ext in ${type_exts[$type]}; do
				printf '%d\t%s\n' "${ext_counts[$ext]}" "$ext"
			done | sort -k1,1rn -k2,2 | cut -f2
		)
		for ext in "${type_ext_list[@]}"; do
			hint_parts+=("\`${ext}\`")
		done
	fi
	hint=""
	for part in "${hint_parts[@]}"; do
		if [ -z "$hint" ]; then
			hint="$part"
		else
			hint+=", $part"
		fi
	done
	printf -- "- %-16s %5d  (%s)\n" "${type}:" "$count" "$hint"
done

# Render the (other) summary line, if anything ended up there.
# Pluralize cleanly.
if [ "$other_kinds" -gt 0 ]; then
	file_word="files"; [ "$other_files" -eq 1 ] && file_word="file"
	kind_word="kinds"; [ "$other_kinds" -eq 1 ] && kind_word="kind"
	echo "- (other): ${other_files} ${file_word} across ${other_kinds} ${kind_word}"
fi

# With --list-unclassified, expand (other) into a nested listing,
# grouped by type so it's easy to see what each rare type is.
# Types are listed in the same order as the lumping loop saw them
# (count descending, then alphabetical), and filenames within each
# type are sorted alphabetically for deterministic output.
if [ "$list_unclassified" -eq 1 ] && [ "${#other_types[@]}" -gt 0 ]; then
	for type in "${other_types[@]}"; do
		count=${type_counts[$type]}
		echo "  - ${type} (${count}):"
		# `type_files` entries are newline-separated; sort them.
		mapfile -t sorted_files < <(printf '%s\n' "${type_files[$type]}" | sort)
		for f in "${sorted_files[@]}"; do
			echo "    - \`${f}\`"
		done
	done
fi