orient-files: shebang detection for executable files
commit 93b6dc49ff1208b6d9b505f3f405c6826295e3cb · parent 9b576736e6
· browse this tree
Ximianth, 2026-06-24 16:51:04 UTC
Adds a second heuristic: for executable regular files, the first 100 bytes are read and the first line is scanned for a recognized interpreter (bash, sh, zsh, dash, ksh, python, node, perl, ruby, lua, awk). Handles both direct-path (#!/bin/bash) and env-dispatched (#!/usr/bin/env python3, with or without -S) shebangs. Restricted to executable files to keep the cost cheap on large trees: almost all source files are non-executable, so the read pass only touches the small subset that's actually a candidate. Per-file precedence: shebang dominates extension when both produce a verdict. A .sh file with a python shebang counts as python. The hint column now lists 'shebang' first when the shebang heuristic contributed to a type, followed by the contributing extensions. When only one heuristic drove the verdict for every file, only that one appears -- e.g. all-extensionless bash scripts show (shebang) without a misleading extension reference. NUL bytes in the first 100 bytes of binaries are stripped before sniffing, since bash command substitution warns on them and a real shebang line can't contain them.
diff --git a/orient-files b/orient-files index 8879078..eae58dc 100755 --- a/orient-files +++ b/orient-files @@ -6,13 +6,13 @@ # # Output: a bullet list, one row per type, sorted by count descending. # Each row has three columns: type name, count, and a hint indicating -# which heuristic(s) matched (currently just the extensions that mapped -# to this type). Types with fewer than 3 files are lumped into a -# trailing "(other)" line so the output stays bounded. +# which heuristic(s) matched. Types with fewer than 3 files are lumped +# into a trailing "(other)" line so the output stays bounded. # # Example output: # - markdown: 25 (`.md`) # - typescript: 1241 (`.ts`) +# - bash: 14 (shebang, `.bash`) # - image: 108 (`.png`, `.jpg`, `.gif`) # - unknown-shell: 7 (`.sh`) # - (other): 9 files across 5 kinds @@ -22,6 +22,18 @@ # through to using the literal extension as the type name, so unfamiliar # projects stay informative without needing the table to be exhaustive. # +# Heuristics: +# - Extension: a small handwritten table mapping `.md` -> markdown etc. +# - Shebang: for executable regular files, the first 100 bytes are +# read and the first line is matched against a known interpreter list +# (bash, sh, zsh, dash, ksh, python, node, perl, ruby, lua, awk). +# Shebang dominates extension when both produce a verdict: a `.sh` +# file with `#!/usr/bin/env python3` counts as python. +# Only executable files get a shebang sniff, which keeps the cost +# bounded -- almost all source files are non-executable. +# The hint column lists which heuristics fired for a given type: it's +# a tally of "reasons we looked", not a derivation trace. +# # Extensionless files (e.g. `LICENSE`, `Makefile`) and dotfiles whose # name carries no further dot (`.gitignore`, `.editorconfig`) are folded # into `(other)` by default. @@ -112,13 +124,34 @@ declare -A ext_to_type=( [.log]=log ) +# Interpreter name (as it appears in a shebang) -> canonical type name. +# Order in the regex matters only for completeness, not for precedence: +# any single file's shebang names one interpreter. +declare -A interp_to_type=( + [bash]=bash + [sh]=sh + [zsh]=zsh + [dash]=dash + [ksh]=ksh + [python]=python [python2]=python [python3]=python + [node]=javascript [nodejs]=javascript + [perl]=perl + [ruby]=ruby + [lua]=lua + [awk]=awk [gawk]=awk [mawk]=awk +) + # `find` permission-denied messages on unrelated subdirs are expected # noise in shared filesystems (e.g. running under /tmp). Filter those # out, but pass any other stderr through unchanged so real problems # still surface. # -# Each found file is awk-classified into either an extension (e.g. `.rs`) -# or the special token `(none)`. Bash then tallies and rolls up to types. +# Each found file is emitted as a TAB-separated triple: +# <ext_or_(none)>\t<basename>\t<x_or_-> +# where the trailing flag marks executable regular files (which then get +# a shebang sniff in a second pass). We need both filename and path-ish +# info to be carried through, so we use -printf with %p (full path) for +# the shebang lookup; the basename comes from %f. mapfile -t classified < <( find "$root" \ \( \ @@ -131,28 +164,31 @@ mapfile -t classified < <( -o -name 'vendor' \ \) -prune \ \) -o \ - \( -type f -printf '%f\n' \) \ + \( -type f \( -executable -printf 'x\t%f\t%p\n' \) \) -o \ + \( -type f -printf '-\t%f\t\n' \) \ 2> >(grep -v 'Permission denied' >&2) \ - | awk ' - { - name = $0 + | awk -F'\t' ' + function classify_ext(name, classname, n, i, idx) { # For classification purposes, drop a leading dot so dotfiles # like `.gitignore` are treated as having no extension. classname = name if (substr(classname, 1, 1) == ".") classname = substr(classname, 2) - - # Find the last dot; everything after it is the extension. n = length(classname) idx = 0 for (i = n; i >= 1; i--) { if (substr(classname, i, 1) == ".") { idx = i; break } } - if (idx == 0 || idx == n) { - # Emit the original filename so we can list it later if requested. - print "(none)\t" name - } else { - print "." substr(classname, idx + 1) - } + if (idx == 0 || idx == n) return "(none)" + return "." substr(classname, idx + 1) + } + { + exec_flag = $1; name = $2; path = $3 + ext = classify_ext(name) + # Output: <ext>\t<basename>\t<exec_flag>\t<path> + # The basename matters only for the extensionless listing under -x. + # The path is only used for the shebang sniff, and only when + # exec_flag == "x"; otherwise it is empty. + print ext "\t" name "\t" exec_flag "\t" path } ' ) @@ -162,32 +198,102 @@ if [ ${#classified[@]} -eq 0 ]; then exit 0 fi -# Partition into per-extension counts and the extensionless bucket. +# Sniff a file's shebang, cheaply. +# Read at most 100 bytes; require `#!` at byte 0; take the first line; +# match a known interpreter token via the basename of whatever follows +# the bang (handles both `#!/bin/bash` and `#!/usr/bin/env bash`). +# Echoes the canonical type name on a match, or nothing on a miss. +sniff_shebang() { + local path=$1 head line interp + # Strip NULs so bash doesn't complain about them in command substitution. + # Binaries often start with a NUL run, and a real shebang line never + # contains NULs anyway. + head=$(head -c 100 -- "$path" 2>/dev/null | tr -d '\000') || return 0 + [[ "$head" == '#!'* ]] || return 0 + line=${head%%$'\n'*} + # Match the rightmost basename-looking token in the shebang. + # Examples it should pick up: + # #!/bin/bash -> bash + # #!/usr/bin/env python3 -> python3 + # #!/usr/bin/env -S bash -> bash + # Trailing flags (e.g. `python3 -u`) are stripped by taking the first + # whitespace-delimited token after the last `/` or after `env`. + # Strip leading `#!` and surrounding whitespace. + line=${line#'#!'} + line=${line## } + # Split on whitespace; walk tokens to find the first that's a known + # interpreter basename (skipping option words like `-S` and the + # literal `env`). + local tok base name + for tok in $line; do + # Skip option flags. + [[ "$tok" == -* ]] && continue + base=${tok##*/} + # Skip the `env` dispatcher; the next token will be the interpreter. + [[ "$base" == "env" ]] && continue + name=${interp_to_type[$base]:-} + if [ -n "$name" ]; then + printf '%s' "$name" + return 0 + fi + # First non-flag, non-env token decided the interpreter (even if + # we don't recognize it). Bail rather than walking into argv. + return 0 + done +} + +# Partition into per-extension counts, the extensionless bucket, and +# shebang-driven reassignments. +# +# Per-type tallies are accumulated directly; for the hint column we +# track which heuristics fired and (for the extension heuristic) which +# extensions contributed. +# `type_counts[type]` -- total files for that type. +# `type_exts[type]` -- space-separated set of contributing extensions. +# `type_shebang[type]` -- count of files whose shebang chose this type. +# `ext_counts[ext]` -- per-extension count, used for sorting the hint +# list within a type (dominant variant first). +declare -A type_counts=() +declare -A type_exts=() +declare -A type_shebang=() declare -A ext_counts=() extensionless_files=() for entry in "${classified[@]}"; do - if [[ "$entry" == '(none) '* ]]; then - extensionless_files+=("${entry#'(none) '}") - else - ext_counts["$entry"]=$((${ext_counts["$entry"]:-0} + 1)) + IFS=$'\t' read -r ext name exec_flag path <<< "$entry" + + # Decide the type for this individual file. + # Shebang dominates extension when both produce a verdict. + shebang_type="" + if [ "$exec_flag" = "x" ] && [ -n "$path" ]; then + shebang_type=$(sniff_shebang "$path") fi -done -# Roll up extensions into types. -# Each type tracks its total count and the set of source extensions that -# contributed (rendered later as the hint column). -# `type_exts` stores a space-separated list, treated as a set: an extension -# only appears once per type, since `ext_counts` is keyed by extension. -declare -A type_counts=() -declare -A type_exts=() -for ext in "${!ext_counts[@]}"; do - type=${ext_to_type[$ext]:-$ext} - type_counts[$type]=$((${type_counts[$type]:-0} + ${ext_counts[$ext]})) - if [ -z "${type_exts[$type]:-}" ]; then - type_exts[$type]=$ext + if [ -n "$shebang_type" ]; then + type=$shebang_type + type_shebang[$type]=$((${type_shebang[$type]:-0} + 1)) + # Don't count this file's extension toward the type's ext set. + # If the extension agreed with the shebang, we still want the row + # to show that extension as a contributing heuristic -- but only + # if some *other* file of the same extension landed here too. + # That happens naturally below when non-shebang files of that + # extension are processed. + elif [ "$ext" = "(none)" ]; then + extensionless_files+=("$name") + continue else - type_exts[$type]+=" $ext" + type=${ext_to_type[$ext]:-$ext} + ext_counts[$ext]=$((${ext_counts[$ext]:-0} + 1)) + if [ -z "${type_exts[$type]:-}" ]; then + type_exts[$type]=$ext + else + # Add to set only if not already present. + case " ${type_exts[$type]} " in + *" $ext "*) ;; + *) type_exts[$type]+=" $ext" ;; + esac + fi fi + type_counts[$type]=$((${type_counts[$type]:-0} + 1)) done # Sort types by count descending, ties broken alphabetically. @@ -225,24 +331,34 @@ done # - Count right-aligned in 5 chars. # - Two spaces, then the hint in parentheses. # -# Per-type extension lists in the hint are sorted by per-extension count -# descending (so the dominant variant appears first, e.g. `.yaml` ahead -# of `.yml`), then alphabetically. This makes output deterministic -# despite bash assoc-array iteration order being unspecified. +# Hint composition: `shebang` appears first if the shebang heuristic +# fired for this type, then the contributing extensions sorted by +# per-extension count descending (dominant variant first), then +# alphabetically. Determinism is important since bash assoc-array +# iteration order is unspecified. for i in "${!shown_types[@]}"; do type=${shown_types[$i]} count=${shown_counts[$i]} - mapfile -t type_ext_list < <( - for ext in ${type_exts[$type]}; do - printf '%d\t%s\n' "${ext_counts[$ext]}" "$ext" - done | sort -k1,1rn -k2,2 | cut -f2 - ) + hint_parts=() + if [ -n "${type_shebang[$type]:-}" ]; then + hint_parts+=("shebang") + fi + if [ -n "${type_exts[$type]:-}" ]; then + mapfile -t type_ext_list < <( + for ext in ${type_exts[$type]}; do + printf '%d\t%s\n' "${ext_counts[$ext]}" "$ext" + done | sort -k1,1rn -k2,2 | cut -f2 + ) + for ext in "${type_ext_list[@]}"; do + hint_parts+=("\`${ext}\`") + done + fi hint="" - for ext in "${type_ext_list[@]}"; do + for part in "${hint_parts[@]}"; do if [ -z "$hint" ]; then - hint="\`${ext}\`" + hint="$part" else - hint+=", \`${ext}\`" + hint+=", $part" fi done printf -- "- %-16s %5d (%s)\n" "${type}:" "$count" "$hint"