Merge branch 'format-doc'
commit 46343bf21dad668aa903338887dd2a219d8ec14a · parent 2169820719 · parent ff2c104989
· browse this tree
Ximianth, 2026-10-19 00:43:45 UTC
diff --git a/bin/patchette b/bin/patchette index a2df97b..5355755 100755 --- a/bin/patchette +++ b/bin/patchette @@ -6,29 +6,12 @@ ## patchette import <target-repo> <patches-dir> ## patchette verify <subject-repo> <patches-dir> ## -## Layout, per branch (folder name = branch name, `/` encoded as `¦` U+00A6): -## STACK `ref`/`base`/`tip` lines, then `series`, then slugs to EOF -## <slug>.patch one file per commit: parents line, headers, framed message, diff +## The on-disk format is specified in docs/hacking/FORMAT.md, and the replay +## algorithm in docs/hacking/REPLAYING.md; those documents are the single +## source of truth, and this tool conforms to them. ## -## The patch file, exactly: -## line 1 `parents <ref>...` (each ref: BASE, or an earlier slug) -## line 2 `message-no-final-newline` (only in that freak case; contains no -## space, so it can never collide with a verbatim `key value` header) -## next lines commit headers verbatim, minus tree/parent -## blank line -## `----` opening flank of the message block -## next lines message, framed per the sigil dialect (see below) -## `----` closing flank -## blank line -## rest diff vs first parent, to EOF (empty for empty commits) -## -## Message framing is exactly invertible and never transforms content bytes. -## Two sigil dialects exist; export emits one (default `quote`), the parser -## accepts both unconditionally (the `----` flanks keep mixed acceptance -## unambiguous: a message line reading `----` is always sigil-framed): -## quote: empty line -> `>`, else `> ` + line -## tab: empty line -> ``, else TAB + line -## A message line in any other shape is a parse error, never a guess. +## Two message-framing sigil dialects exist; export emits one (default `quote`), +## and the parser accepts both unconditionally. ## ## Fidelity and fail-closed behavior outrank convenience: parse errors never ## guess, every refusal says what it refused and why, and a failed import @@ -126,11 +109,37 @@ parse_stack() { *) die "$f: unrecognized line: $line" ;; esac done <"$f" - [[ -n $STACK_REF && -n $STACK_BASE && -n $STACK_TIP && $mode == series ]] || - die "$f: missing ref, base, tip, or series section" + # tip is optional: a hand-made STACK has no expectable value for it + # until the branch has been built. + [[ -n $STACK_REF && -n $STACK_BASE && $mode == series ]] || + die "$f: missing ref, base, or series section" [[ $STACK_REF == refs/heads/* ]] || die "$f: ref is not under refs/heads/: $STACK_REF" } +## Check folder-level concordance between a STACK's series and the *.patch +## files beside it. Any discordance is an error for every reader: +## a series entry naming no file would fail replay anyway, and an orphan +## patch file is silently inert -- nothing would ever read it. +## (Duplicate series entries are caught earlier, in parse_stack.) +## +## Expects parse_stack to have run for this folder. +## +## - @param $1 branch folder path +validate_branch_dir() { + local dir=$1 slug f + declare -A listed=() + for slug in "${STACK_SERIES[@]}"; do + listed[$slug]=1 + [[ -f $dir/$slug.patch ]] || die "$dir/STACK: series lists $slug, but $slug.patch is missing" + done + for f in "$dir"/*.patch; do + [[ -f $f ]] || continue # unmatched glob + slug=${f##*/} + slug=${slug%.patch} + [[ -n ${listed[$slug]:-} ]] || die "$f: orphan patch file, not listed in the STACK series" + done +} + ## Replay one branch folder's series into the current repo's object store. ## ## Expects parse_stack to have run for this folder, cwd to be a git repo @@ -148,7 +157,6 @@ replay_series() { for slug in "${STACK_SERIES[@]}"; do f="$dir/$slug.patch" - [[ -f $f ]] || die "$dir/STACK: series lists $slug, but $slug.patch is missing" # Parse the patch file: a strict little state machine over its prelude. # The diff section is everything after the blank line that follows the @@ -161,27 +169,37 @@ replay_series() { start) [[ $line == parents\ * ]] || die "$f: line 1 is not a parents line" pline=$line - state=flagmaybe - ;; - flagmaybe) - if [[ $line == message-no-final-newline ]]; then - nofinal=1 - state=headers - continue - fi state=headers - ;& + ;; headers) if [[ -z $line ]]; then state=msgopen + elif [[ $line != *' '* ]]; then + # Headers are `key value`, and continuation lines begin with + # a space, so every header line contains one. + die "$f: malformed header line (no space): $line" else headers+=("$line") fi ;; msgopen) [[ $line == '----' ]] || die "$f: expected opening ---- flank, got: $line" - state=message + state=directives ;; + directives) + # Bare directive lines may sit directly after the opening flank, + # before any message content. Anything unrecognized and equally + # bare in this position is an error; anything message-shaped + # means the content has begun. + if [[ $line == message-no-final-newline ]]; then + nofinal=1 + continue + fi + if [[ -n $line && $line != '----' && $line != '>' && $line != '> '* && $line != $'\t'* ]]; then + die "$f: unrecognized directive line: $line" + fi + state=message + ;& message) # Both sigil dialects are accepted here, unconditionally. if [[ $line == '----' ]]; then @@ -241,10 +259,14 @@ replay_series() { done } -## List the branch folders of a patches dir, failing closed on strangers. +## List the branch folders of a patches dir, failing closed on strangers, +## and validate every folder's concordance up front: discordance is a static +## property of the patches dir, so it's refused before any repo is touched +## (and in import's case, before any ref could be created). ## ## - @param $1 patches dir -## - @state BRANCH_DIRS[] -- absolute paths, each guaranteed to hold a STACK +## - @state BRANCH_DIRS[] -- absolute paths, each holding a validated STACK +## - @state STACK_* -- clobbered; callers re-run parse_stack per folder branch_dirs() { local patches=$1 dir BRANCH_DIRS=() @@ -252,6 +274,8 @@ branch_dirs() { dir=${dir%/} [[ -d $dir ]] || continue # unmatched glob [[ -f $dir/STACK ]] || die "$dir has no STACK file; not a patchette branch folder" + parse_stack "$dir/STACK" + validate_branch_dir "$dir" BRANCH_DIRS+=("$dir") done [[ ${#BRANCH_DIRS[@]} -gt 0 ]] || die "no branch folders found in $patches" @@ -344,11 +368,11 @@ cmd_export() { { printf 'parents %s\n' "${parents[*]}" + sed -n '1,/^$/p' "$raw" | grep -vE '^(tree|parent) ' | sed '/^$/d' + printf '\n----\n' if [[ -s $msg && $(tail -c 1 "$msg") != '' ]]; then printf 'message-no-final-newline\n' fi - sed -n '1,/^$/p' "$raw" | grep -vE '^(tree|parent) ' | sed '/^$/d' - printf '\n----\n' while IFS= read -r mline || [[ -n $mline ]]; do if [[ -z $mline ]]; then if [[ $sigil == quote ]]; then printf '>\n'; else printf '\n'; fi @@ -390,7 +414,7 @@ cmd_import() { # Empty old-value makes must-not-exist atomic: no clobbering, even racily. git update-ref "$STACK_REF" "$REPLAY_TIP" "" echo "import: created $STACK_REF at $REPLAY_TIP" - if [[ $REPLAY_TIP != "$STACK_TIP" ]]; then + if [[ -n $STACK_TIP && $REPLAY_TIP != "$STACK_TIP" ]]; then echo "import: note: replayed tip differs from recorded tip $STACK_TIP (patches edited since the snapshot?)" fi done @@ -398,8 +422,7 @@ cmd_import() { cmd_verify() { [[ $# -eq 2 ]] || die "usage: patchette verify <subject-repo> <patches-dir>" - local subject patches dir f slug bad=0 - declare -A listed + local subject patches dir bad=0 subject=$(absdir "$1") patches=$(absdir "$2") branch_dirs "$patches" @@ -415,27 +438,15 @@ cmd_verify() { git cat-file -e "$STACK_BASE^{commit}" 2>/dev/null || die "recorded base $STACK_BASE is not present in $subject" replay_series "$dir" - if [[ $REPLAY_TIP == "$STACK_TIP" ]]; then + if [[ -z $STACK_TIP ]]; then + echo "verify: note: $STACK_REF replays cleanly to $REPLAY_TIP, but records no tip to cross-check against" + elif [[ $REPLAY_TIP == "$STACK_TIP" ]]; then echo "verify: ok: $STACK_REF reproduces recorded tip $STACK_TIP" else echo "verify: MISMATCH: $STACK_REF replays to $REPLAY_TIP, but records tip $STACK_TIP" echo "verify: either the snapshot was unfaithful, or the patches were edited since (patches repo is ahead of its snapshot)" bad=1 fi - - # Orphan patch files are silently inert at replay; nothing else would - # ever read them, so flagging them is specifically verify's job. - listed=() - for slug in "${STACK_SERIES[@]}"; do listed[$slug]=1; done - for f in "$dir"/*.patch; do - [[ -f $f ]] || continue # unmatched glob - slug=${f##*/} - slug=${slug%.patch} - if [[ -z ${listed[$slug]:-} ]]; then - echo "verify: orphan: $f is not listed in STACK" - bad=1 - fi - done done [[ $bad -eq 0 ]] || exit 1 diff --git a/docs/hacking/FORMAT.md b/docs/hacking/FORMAT.md new file mode 100644 index 0000000..96950bd --- /dev/null +++ b/docs/hacking/FORMAT.md @@ -0,0 +1,200 @@ +the Patchette file formats +========================== + +This document is the single source of truth for patchette's on-disk format: +both the content of the files, and how they're arranged on the filesystem. + +This description aims to be simple and implementation-agnostic: we hope +a correct implementation of export and replay can be written from this document alone, in any language. + + +Goals +----- + +A quick recap on what we want from these files: + +- **Lossless roundtrip of git commits.** + - All git commit metadata must be persisted. Messages, authorship, dates, everything. + - All git commit relationships must be persisted. This includes merge commits. + - The patch files replayed on to the same base commit in a repo should result in + the same branch and same SHA of its tip as was stored. +- **Reviewable evolution.** + - The layout is plain text files designed to diff when the patches change, + and stay unaffected whenever possible. The diffs should be meaningful. + - To this end: export is deterministic, and must depend only on the commits + (and never on any git configuration, attributes, or other distracting chaff). +- **Editable by hand.** + - While patchette is meant to allow the users to work within repo when making changes, + we also absolute want to support the edit of patches directly, with ordinary tools, too. + Therefore the formats must be easy to read, modify, and must also be easy to heal. + + +Directory Layout +---------------- + +A *patches dir* contains one directory per tracked branch, +and a series of patch files and one control file in each one. +This can look roughly like the following: + +``` + <patches-dir>/ + topic¦fancy/ + STACK + fix-the-frobnicator.patch + fix-the-frobnicator-2.patch + merge-sprocket-work.patch + ... + docs-polish/ + STACK + woggle-the-borogrove.patch + ... +``` + +### branch directory names + +Directory names are derived from the branch name, with every `/` replaced by `¦` (U+00A6 BROKEN BAR). +(The directory name is a convenience; the `ref` line inside the `STACK` file is the more authoritative value.) + +### patch file names + +Patch file names are derived from commit subjects. + +A sluggifcation is used: the subject is transformed lowercase; +every run of non-alphanumerics becomes one `-`; +dashes trimmed at both ends; +the result is truncated to 40 chars (re-trimmed); +and may never be empty (fallback: `patch`). +Collisions within one series take `-2`, `-3`, ... +in series order. + +Patch file names may "change" if the "same" branch is exported +from a git repo after a rebase or other edits that change the commit subject line. +A reworded subject will likely show up as a file rename +(as git itself would describe it, when tracking the patch files, based on similarity heuristics -- probably; +depending on how much else changed during the rebase). +This may feel a little dissatisfying, but it's unclear what other behavior could be better without inventing concepts of identity that git just doesn't have. + + +The STACK file +-------------- + +The `STACK` file contains the metadata about a branch +(its name, the commit in the upstream it was based on, etc) +and the ordering of commits within it. + +It uses a simple line-oriented format which looks like the following: + +``` +ref refs/heads/topic/fancy +base 5b0130cfd2153bcdb4a6d7ec356e7d805018afc9 +tip cc261dc9f0b878a78e2b80aeb60de397205f61ad +series +fix-the-frobnicator +fix-the-frobnicator-2 +[...addntl commit names...] +``` + +Several pieces of `key value` format data start the file, +then a literal `series` line denotes the end of that section, +and then one patch file name is per line, until EOF. + +Of the key value entries: + +- `ref`: is the full ref name of the tracked branch. +- `base`: the commit SHA of the upstream base this stack was last wrangled against. + (If rebuilding a branch in a fresh clone of the upstream repo from the patch files, + this is the commit that will be checked out before beginning to replay the patches.) +- `tip`: the commit SHA that's expected when this stack is applied to the base commit. + This is saved when a branch is exported from a repo into patch files. + It's optional: when present, it's useful for verification that rebuilding a branch from patches worked correctly. + It may also be disregarded in some workflows: e.g., when hand-editing patch files, + there's no expectable value for tip until the branch has been built. + +In the `series` section: + +- each line is a patch name. These match `^[a-z0-9-]+$`, + and each names a `<slug>.patch` file in the same directory. + +The order of patch names is topological. +While parent information is also stored in the individual patch files, +and thus an overall toposort could be recomputed by reading all the patch files, +we insist that the patch list in the `STACK` file must already be a valid toposort; +tools may (and should) error if it is not. +Duplicate patch names; patch names that refer to no file; and `*.patch` files with no mention +in the series section are all errors. + + +The patch file +-------------- + +Each `*.patch` file looks like the following: + +``` +parents merge-sprocket-work +author Ada Downstream <ada@downstream.example> 1700000480 +0000 +committer Ada Downstream <ada@downstream.example> 1700000480 +0000 +change-id kpqxwvuzyntsmrlokqjihvgzfyexwdcu + +---- +> Teach gizmo to hum +> +> The hum is load-bearing. +> Do not ask why. +---- + +diff --git a/frobnicator.txt b/frobnicator.txt +[...typical diff format...] +``` + +Commit structure and metadata are at the top; +the commit message follows (with substantial demarcations, since it can be freetext); +the diff follows that, and continues to the end of file. + +In detail: + +- **Commit graph structure**: + - `parents <name>...` -- one or more references, space-separated. + - Each `<name>` is either `BASE` (the STACK's base commit) or the name of another patch in the STACK. +- **Commit metadata**: + - The commit's raw headers, verbatim, in the same order as git handles them... modulo the `tree` and `parent` (since we handle that data in other ways). + - Each line is `key value` separated by a space. + - Yes, everything git handles, verbatim. + - So this can include multi-line headers -- e.g. `gpgsig`, `mergetag` -- continuation lines for these begin with a space. + - And other headers unknown to Patchette (e.g. `jj`'s `change-id` rides through this way). + - This block ends at the first empty line. + (Git headers cannot contain an empty line, so this is unambiguous.) +- **Commit message**: + - Flanked by literal `----` lines. + - Directive lines may appear directly after the opening `----`, before any message content. + These are bare lines (no prefix); any unrecognized line in this position is an error. + Currently defined: + - `message-no-final-newline` -- declares that the message does not end with a newline character. + (Possible at the git object level, though porcelain tools essentially never produce it. + But lossless means lossless.) + - Each line of body between the `----` boundary markers carries a prefix. + (This ensures that commit messages containing a line of dashes cannot produce a fuss.) + - Two prefix styles are supported: {`> `+line} and {`\t`+line}. + - In both styles, a line with no content skips all whitespace (meaning a `>` is alone, with no trailing space, and a `\t` is skipped entirely). + (This helps the format be produce fewer warnings about trailing whitespace under typical conditions.) + - Note that trailing whitespace in content lines is permitted. + (If patch files are edited by users, their editors may mangle this; + however, there's little we can do about that.) + - One blank line is expected after the closing `----` flank. + (This is mostly just for visual consistency.) +- **Diff section**: + - Continues for the rest of the file until EOF. + - May be empty. (Empty commits are valid.) + - This is the commit's changes vs its *first* parent. + +### diff generation + +Diff generation is: + +``` +git diff-tree -r -p --full-index --binary --no-renames --diff-algorithm=patience <parent1> <commit> +``` + +...and must be run in an environment where no configuration or attributes can reach the diff machinery -- e.g., no global/system/repo-local config, and no gitattributes from any checkout. + +- `--no-renames`: because rename detection changes representation when content drifts, + which manufactures phantom diffs in the patches dir; delete+add is bigger but stable. diff --git a/docs/hacking/REPLAYING.md b/docs/hacking/REPLAYING.md new file mode 100644 index 0000000..ee5bc68 --- /dev/null +++ b/docs/hacking/REPLAYING.md @@ -0,0 +1,26 @@ +Replaying: rebuilding branches from patch files +=============================================== + +The FORMAT document describes what the files *are*; +this one describes how a tool turns them back into git commits, exactly. +Any implementation following these steps reproduces the stored branch SHA-exact. + +For each patch name in series order, maintaining a map from reference to replayed SHA +(seeded with `BASE` -> the STACK's base): + +1. Parse the patch file. Resolve each `parents` reference through the map; + an unknown reference is an error (series order guarantees parents were replayed first). +2. Read the first parent's tree into a scratch index (no worktree is required, ever). +3. If the diff section is non-empty, apply it to that index + (`git apply --cached --binary --whitespace=nowarn`; the explicit whitespace flag + keeps a target repo's own `apply.whitespace` from rewriting or refusing recorded bytes). + An apply failure is an error: the recorded base is the only supported target. +4. Write the resulting tree. +5. Reassemble the raw commit object: `tree` line, one `parent` line per resolved parent, + then the stored header block verbatim, a blank line, and the message bytes + (dropping the final newline if the `message-no-final-newline` directive was declared). + Hash and store it (`git hash-object -t commit -w`). Record name -> SHA in the map. + +The final SHA is the replayed branch tip. +When the STACK records a `tip`, verification compares against it; +a STACK without one can still be replayed, just not cross-checked. diff --git a/tests/test-failclosed.sh b/tests/test-failclosed.sh index ed27b1a..6c77dd5 100755 --- a/tests/test-failclosed.sh +++ b/tests/test-failclosed.sh @@ -44,6 +44,27 @@ sed -i 's/> Do not ask why./| Do not ask why./' "$TAMPER/$fancy/teach-gizmo-to-h EXPECT refusal "a malformed message line is a parse error" "malformed message line" -- \ "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" +# An unrecognized bare line in directive position (directly after the +# opening flank) is an error. +clone_layout baddirective +sed -i '0,/^----$/s//----\nmystery-directive/' "$TAMPER/$fancy/note-to-self.patch" +EXPECT refusal "an unknown directive line is an error" "unrecognized directive" -- \ + "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" + +# A directive is only legal before message content; after content has begun, +# the same bare line is just a malformed message line. +clone_layout latedirective +sed -i 's/^> Note to self$/> Note to self\nmessage-no-final-newline/' "$TAMPER/$fancy/note-to-self.patch" +EXPECT refusal "a directive after message content is a parse error" "malformed message line" -- \ + "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" + +# The directive's pre-spec position (bare line in the header area) must fail +# loudly, not be swallowed as a verbatim header into the replayed commit. +clone_layout olddirective +sed -i '1a message-no-final-newline' "$TAMPER/$fancy/note-to-self.patch" +EXPECT refusal "a bare line in the header area is a parse error" "malformed header line" -- \ + "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" + # An unknown parent reference is an error. clone_layout badparent sed -i '1s/.*/parents no-such-slug/' "$TAMPER/$fancy/add-sprocket-docs.patch" @@ -56,11 +77,25 @@ echo "note-to-self" >>"$TAMPER/$fancy/STACK" EXPECT refusal "a duplicate series entry is an error" "duplicate series entry" -- \ "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" -# An orphan patch file (nothing would ever read it) is flagged by verify. +# Folder discordances are errors for every reader: an orphan patch file +# (nothing would ever read it) and a series entry naming no file must be +# refused by verify and import alike. clone_layout orphan cp "$TAMPER/$fancy/note-to-self.patch" "$TAMPER/$fancy/stray.patch" -EXPECT refusal "an orphan patch file is flagged by verify" "orphan" -- \ +EXPECT refusal "an orphan patch file is refused by verify" "orphan" -- \ + "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" +git clone -q "$work/corpus/upstream" "$work/orphantarget" +EXPECT refusal "an orphan patch file is refused by import" "orphan" -- \ + "$PATCHETTE" import "$work/orphantarget" "$TAMPER" +EXPECT equal "the refused import created no refs" -- \ + "$(git -C "$work/orphantarget" for-each-ref --format='%(refname)' 'refs/heads/*' --exclude refs/heads/main)" "" + +clone_layout missingfile +rm "$TAMPER/$fancy/note-to-self.patch" +EXPECT refusal "a series entry naming no file is refused by verify" "note-to-self.patch is missing" -- \ "$PATCHETTE" verify "$work/corpus/dev" "$TAMPER" +EXPECT refusal "a series entry naming no file is refused by import" "note-to-self.patch is missing" -- \ + "$PATCHETTE" import "$work/orphantarget" "$TAMPER" # Export never deletes a folder it didn't write. mkdir -p "$work/notours/$fancy" diff --git a/tests/test-optional-tip.sh b/tests/test-optional-tip.sh new file mode 100755 index 0000000..ecbda59 --- /dev/null +++ b/tests/test-optional-tip.sh @@ -0,0 +1,29 @@ +#!/bin/bash +## The tip line in STACK is optional: export always writes it, but a hand-made +## STACK without one must replay fine. Verify then does everything except the +## tip comparison and says so; import simply skips its advisory comparison. +set -euo pipefail +cd "$(dirname "$0")" +source ./lib.sh + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +make_corpus "$work/corpus" +source "$work/corpus/FACTS" + +"$PATCHETTE" export "$work/corpus/dev" "$work/patches" "$base" topic/fancy >/dev/null +sed -i '/^tip /d' "$work/patches/topic¦fancy/STACK" + +"$PATCHETTE" verify "$work/corpus/dev" "$work/patches" >"$work/verify.out" +EXPECT success "verify passes on a tipless STACK, noting it couldn't cross-check" -- \ + grep -q "records no tip to cross-check" "$work/verify.out" + +git clone -q "$work/corpus/upstream" "$work/fresh" +"$PATCHETTE" import "$work/fresh" "$work/patches" >"$work/import.out" +EXPECT equal "import reproduces the branch from a tipless STACK" -- \ + "$(git -C "$work/fresh" rev-parse refs/heads/topic/fancy)" "$tip_topic_fancy" +EXPECT equal "import stays silent about tips it cannot compare" -- \ + "$(grep -c 'note' "$work/import.out")" "0" + +exit "$RESULT"