diff --git a/firmware/src/git_sync.rs b/firmware/src/git_sync.rs index 2a0d2c6..fdd3ac3 100644 --- a/firmware/src/git_sync.rs +++ b/firmware/src/git_sync.rs @@ -17,15 +17,23 @@ //! notes. A `/sd/repo` that isn't a valid repo is a provisioning error //! (`just init`), surfaced as such, not papered over. //! 3. **No synthetic content.** The spike appended a marker line; here the -//! editor has already saved the user's `notes.md` before `:sync` signals us, -//! so we just stage + commit + push what's on disk. +//! editor has already saved the user's buffers before `:sync` signals us, +//! so we just commit + push what's on disk. +//! 4. **The commit is an O(depth) TreeBuilder splice, not an index pass.** +//! The request carries the repo-relative paths saved/deleted since the last +//! confirmed publish (`Storage`'s journaled dirty set); `stage_and_commit` +//! patches exactly those onto HEAD's tree. The index pipeline it replaced +//! (`add_all` → `index.write` → `write_tree`) is O(N_tree) and measured up +//! to 611 s on the real 1179-file / 570 MB-pack clone — see +//! docs/tradeoff-curves/sync-commit-staging.md for the whole trail. //! //! Runs on a dedicated 96 KB thread (libgit2's init→push chain nests ~67 KB of //! `GIT_PATH_MAX` stack buffers — see git_push.rs / postmortem #3). Config is //! baked at build time (`TW_*`, ADR-007: v0.1 device config is compiled in). use std::cell::RefCell; -use std::path::Path; +use std::collections::BTreeSet; +use std::fs; use std::rc::Rc; use std::sync::mpsc::{Receiver, Sender}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -39,8 +47,8 @@ use esp_idf_svc::sntp::{EspSntp, SyncStatus}; use esp_idf_svc::sys; use esp_idf_svc::wifi::{BlockingWifi, EspWifi}; use git2::{ - CertificateCheckStatus, Commit, Cred, CredentialType, FetchOptions, IndexAddOption, - PushOptions, RemoteCallbacks, Repository, Signature, + CertificateCheckStatus, Commit, Cred, CredentialType, FetchOptions, ObjectType, Oid, + PushOptions, RemoteCallbacks, Repository, Signature, Tree, }; use crate::net::connect_wifi; @@ -73,10 +81,15 @@ const SNTP_TIMEOUT: Duration = Duration::from_secs(20); /// now also runs here, but it's shallow next to libgit2's path-buffer nesting. pub const GIT_STACK: usize = 96 * 1024; -/// A request to publish. The note is already saved to `/sd/repo/notes.md` by the -/// UI task before this is sent, so the request carries no payload (a future -/// multi-file publish can grow one). -pub struct PublishRequest; +/// A request to publish. The UI task has already saved every dirty buffer to +/// the card before sending this; `paths` is `Storage::take_dirty`'s snapshot — +/// the repo-relative paths saved or `:delete`d since the last confirmed +/// publish. The working tree stays the source of truth: at commit time a path +/// that exists on the card is spliced into the tree from disk, a missing one +/// is spliced out. An unchanged path is a no-op, so over-reporting is safe. +pub struct PublishRequest { + pub paths: BTreeSet, +} /// Result of a publish attempt, sent back to the UI task for the snackbar. The /// detailed error always goes to the serial log; the panel gets a short line. @@ -103,6 +116,22 @@ pub fn run_git_service( rx: Receiver, tx: Sender, ) { + // Process-global libgit2 tuning, once, before any repo work. The 32-bit + // defaults (32 MB window / 256 MB mapped budget, mwindow.c) would + // git__malloc past PSRAM on the first pack access of the real 570 MB-pack + // clone; these are the bench-proven values (git_bench), and ~1.9 MB of + // windows stays live during git ops — the p_mmap emulation (esp_map.c) + // relies on this mapped limit being real. + // SAFETY: set on the git thread before any Repository is opened. + unsafe { + if let Err(e) = git2::opts::set_mwindow_size(256 * 1024) { + log::error!("set_mwindow_size failed ({e}); first pack access may OOM"); + } + if let Err(e) = git2::opts::set_mwindow_mapped_limit(4 * 1024 * 1024) { + log::error!("set_mwindow_mapped_limit failed ({e}); first pack access may OOM"); + } + } + // Lazily initialised on the first request, then reused across publishes. let mut wifi: Option>> = None; let mut modem = Some(modem); @@ -110,7 +139,7 @@ pub fn run_git_service( let mut clock_synced = false; let mut tls_ready = false; - while rx.recv().is_ok() { + while let Ok(req) = rx.recv() { let outcome = publish_cycle( &sys_loop, &mut wifi, @@ -118,6 +147,7 @@ pub fn run_git_service( &mut nvs, &mut clock_synced, &mut tls_ready, + &req.paths, ); let msg = match outcome { Ok(o) => o, @@ -143,11 +173,22 @@ fn publish_cycle( nvs: &mut Option, clock_synced: &mut bool, tls_ready: &mut bool, + paths: &BTreeSet, ) -> Result { if REMOTE_URL.is_empty() || GH_USER.is_empty() || PAT.is_empty() || WIFI_SSID.is_empty() { bail!("git config missing — set TW_WIFI_SSID / TW_REMOTE_URL / TW_GH_USER / TW_PAT in firmware/.env and rebuild"); } + // Nothing recorded dirty and origin's tracking ref already has HEAD: this + // `:sync` has nothing to do — say so without touching the radio (~150 ms + // instead of a Wi-Fi + TLS round). A stranded local commit (committed but + // never pushed, e.g. a push that failed mid-air) makes the check false and + // takes the full path below, where publish_once pushes it. + if paths.is_empty() && remote_current().unwrap_or(false) { + log::info!(":sync — no dirty paths and origin has HEAD; up to date, radio untouched"); + return Ok(PublishOutcome::UpToDate); + } + // Phases are timed so a cold :sync reports where the seconds go. Wi-Fi, clock // and TLS run only on the first sync of a session; a warm sync skips them, so // they read 0 ms and the total collapses to just publish(fetch+commit+push). @@ -186,7 +227,7 @@ fn publish_cycle( } let t_publish = Instant::now(); - let outcome = publish_once()?; + let outcome = publish_once(paths)?; log::info!( ":sync timing — wifi {wifi_ms}ms, clock {clock_ms}ms, tls {tls_ms}ms, publish(commit+push) {}ms, total {}ms", t_publish.elapsed().as_millis(), @@ -205,15 +246,16 @@ fn publish_cycle( /// /// Never clones or wipes: a `/sd/repo` that isn't a valid repo is a provisioning /// error, surfaced as such. -fn publish_once() -> Result { - log::info!("publish started — free heap {}", free_heap()); +fn publish_once(paths: &BTreeSet) -> Result { + log::info!( + "publish started — {} dirty path(s), free heap {}", + paths.len(), + free_heap() + ); let repo = Repository::open(REPO_DIR).with_context(|| { format!("opening git repo at {REPO_DIR} — provision the card with a clone (just init) whose origin is your remote") })?; - let Some(mut oid) = stage_and_commit(&repo)? else { - return Ok(PublishOutcome::UpToDate); - }; let branch = repo .head()? .shorthand() @@ -221,17 +263,46 @@ fn publish_once() -> Result { .to_string(); let refspec = format!("refs/heads/{branch}:refs/heads/{branch}"); - // Optimistic push. A non-fast-forward rejection means the remote moved under - // us: reconcile onto origin and replay the note on the new tip, then retry - // once. reconcile_onto_origin mixed-resets, so the just-saved note survives in - // the working tree and stage_and_commit lands it on top of origin. - if let Err(first) = try_push(&repo, &refspec) { - log::warn!("push rejected ({first}); reconciling onto origin and replaying the note"); + let mut oid = match stage_and_commit(&repo, paths)? { + Some(oid) => oid, + None => { + // Nothing new to commit. Usually genuinely up to date — but a + // previous cycle may have committed and then failed to push, + // stranding a local-only commit (the old add_all path silently + // never retried those). Push whenever origin's tracking ref + // doesn't already have HEAD. + let head = repo.head()?.peel_to_commit()?.id(); + if tracking_tip(&repo, &branch) == Some(head) { + return Ok(PublishOutcome::UpToDate); + } + log::info!( + "tree unchanged but origin/{branch} lacks HEAD {} — pushing the stranded commit", + short(head) + ); + head + } + }; + + // Optimistic push. A non-fast-forward *rejection* means the remote moved + // under us: reconcile onto origin and replay the dirty paths on the new + // tip, then retry once (reconcile_onto_origin soft-resets — ref move only — + // so the notes stay on the card and stage_and_commit splices them on top of + // origin). A transport-level failure is surfaced as-is: its fetch would die + // the same way, and the commit is safe locally — the stranded-commit check + // above pushes it once the transport works again. + if let Err(failure) = try_push(&repo, &refspec) { + let rejection = match failure { + PushFailure::Rejected(msg) => msg, + PushFailure::Other(e) => return Err(e), + }; + log::warn!("push rejected ({rejection}); reconciling onto origin and replaying the note"); reconcile_onto_origin(&repo, &branch).context("reconciling after a rejected push")?; - match stage_and_commit(&repo)? { + match stage_and_commit(&repo, paths)? { Some(replayed) => { oid = replayed; - try_push(&repo, &refspec).context("push after reconcile")?; + try_push(&repo, &refspec) + .map_err(PushFailure::into_error) + .context("push after reconcile")?; } // The note was already on origin (nothing to replay) — treat as done. None => { @@ -249,65 +320,59 @@ fn publish_once() -> Result { Ok(PublishOutcome::Pushed(short(oid))) } -/// Stage the working tree and commit it on top of the current branch tip. -/// Returns the new commit id, or `None` when the tree already matches the parent -/// (nothing to publish). Called on the first attempt and again to replay the note -/// after a reconcile. +/// Build the commit for `paths` as an O(depth) TreeBuilder splice onto HEAD's +/// tree and return the new commit id — or `None` when the result matches the +/// parent (nothing to publish). Called on the first attempt and again to +/// replay the dirty paths after a reconcile. /// -/// Staging is `add --all` **plus** `add -u`, which together equal `git add -A`. -/// `add_all` stages new + modified files; `update_all` re-syncs already-tracked -/// entries to the working tree, which is what actually removes an index entry -/// whose file was deleted. Spike 14 found `add_all` alone did **not** stage a -/// `:delete`d file's removal on this libgit2 build (the tree came back unchanged, -/// so the publish was a silent no-op), so the `update_all` pass is load-bearing, -/// not belt-and-braces — do not drop it. +/// This replaces the index pipeline (`add_all` → `index.write` → `write_tree`), +/// which is O(N_tree) and cannot run on the real 1179-file / 570 MB-pack clone: +/// `index.write`'s racy-clean pass re-hashes ~every entry on FAT's 2 s mtimes +/// (measured up to **611 s**), and even the index-free `read_tree` walk was +/// 77 s. The splice reads and writes only the dirty paths' ancestor chains — +/// O(depth × dirty), flat in repo size, **~2–2.8 s measured on the real +/// clone** — and carries every untouched entry (including the ~150 MB of +/// images) forward by OID without ever opening it. Trail + bench numbers: +/// docs/tradeoff-curves/sync-commit-staging.md. /// -/// Both run a per-path filter that drops macOS AppleDouble sidecars (`._name`) -/// and `.DS_Store` that Finder/Spotlight sprinkle onto the FAT card whenever it's -/// mounted on a Mac — without it, a blind add --all sweeps them into the commit -/// (it did once: 07d87772 shipped `._.git`, `._README.md`, `._notes.md`). -/// Filtering here fixes it for *every* repo at the device level, so no per-repo -/// `.gitignore` is needed. -fn stage_and_commit(repo: &Repository) -> Result> { - let mut index = repo.index().context("opening index")?; - let mut skip_macos_cruft = |path: &Path, _matched: &[u8]| -> i32 { - match path.file_name().and_then(|n| n.to_str()) { - Some(name) if name.starts_with("._") || name == ".DS_Store" => 1, // skip - _ => 0, // add - } - }; - // Split the commit window into its sub-phases so we can tell the FAT - // working-tree *walk* (`add_all`/`update_all` stat every file over SPI) apart - // from the FAT object *writes* (index/tree/commit). This decides whether - // explicit-path staging is worth it: the walk is O(tree size) and avoidable - // (the editor knows the dirty paths), the writes are O(churn) and a floor. - // See docs/tradeoff-curves/sync-commit-staging.md. - let t_walk = Instant::now(); - index - .add_all(["*"], IndexAddOption::DEFAULT, Some(&mut skip_macos_cruft)) - .context("staging new/modified (add --all)")?; - // Stage deletions: update_all removes index entries whose working-tree file is - // gone. add_all does not do this reliably here (Spike 14), so this is required. - index - .update_all(["*"], Some(&mut skip_macos_cruft)) - .context("staging deletions (add -u)")?; - let walk_ms = t_walk.elapsed().as_millis(); - - let t_index = Instant::now(); - index.write().context("writing index")?; - let index_ms = t_index.elapsed().as_millis(); - - let t_tree = Instant::now(); - let tree = repo.find_tree(index.write_tree().context("writing tree")?)?; - let tree_ms = t_tree.elapsed().as_millis(); - - // Commit on top of the current branch tip (None on an empty/unborn remote). - let t_parent = Instant::now(); +/// The working tree is the source of truth: a recorded path that exists on the +/// card is spliced in from disk, a missing one is spliced out (a `:delete`). +/// Unrecorded paths are never visited — so Finder cruft (`._*`, `.DS_Store`) +/// on the FAT card can no longer ride into a commit the way it once did with +/// `add_all` (07d87772), and the old cruft filter is gone with the walk. +fn stage_and_commit(repo: &Repository, paths: &BTreeSet) -> Result> { + // Commit on top of the current branch tip (None on an empty/unborn remote, + // where the splice starts from an empty base and makes a parentless commit). let parent = repo.head().ok().and_then(|h| h.peel_to_commit().ok()); - let parent_ms = t_parent.elapsed().as_millis(); - log::info!( - "commit split — walk(add_all+update_all) {walk_ms}ms, index.write {index_ms}ms, write_tree {tree_ms}ms, parent-load {parent_ms}ms" - ); + let base = match &parent { + Some(c) => Some(c.tree().context("loading HEAD tree")?), + None => None, + }; + + let t_splice = Instant::now(); + let mut tree = base; + for path in paths { + let parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect(); + if parts.is_empty() { + continue; + } + let blob = match fs::read(format!("{REPO_DIR}/{path}")) { + Ok(bytes) => Some( + repo.blob(&bytes) + .with_context(|| format!("writing blob for {path}"))?, + ), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, // deleted → splice out + Err(e) => return Err(e).with_context(|| format!("reading {path}")), + }; + let spliced = splice(repo, tree.as_ref(), &parts, blob) + .with_context(|| format!("splicing {path}"))?; + tree = Some(repo.find_tree(spliced).context("loading spliced tree")?); + } + let Some(tree) = tree else { + return Ok(None); // unborn branch and nothing dirty — nothing to commit + }; + let splice_ms = t_splice.elapsed().as_millis(); + if let Some(p) = &parent { if p.tree_id() == tree.id() { log::info!("nothing to publish — tree unchanged @ {}", short(p.id())); @@ -322,20 +387,111 @@ fn stage_and_commit(repo: &Repository) -> Result> { let oid = repo .commit(Some("HEAD"), &sig, &sig, &message, &tree, &parents) .context("creating commit")?; - let commit_ms = t_commit.elapsed().as_millis(); log::info!( - "commit split — commit-obj {commit_ms}ms; committed {} — free heap {}", + "commit split — splice {splice_ms}ms ({} path(s)), commit-obj {}ms; committed {} — free heap {}", + paths.len(), + t_commit.elapsed().as_millis(), short(oid), free_heap() ); Ok(Some(oid)) } +/// Return a new tree = `base` with `path` set to `blob` (`Some` inserts or +/// replaces, `None` removes). Recurses down the path's subtree chain: reads +/// ~depth tree objects and writes ~depth new ones, leaving every sibling entry +/// untouched (carried by OID — never opened). A missing intermediate directory +/// is synthesized on the way down; a directory emptied by a remove is pruned +/// on the way up rather than left behind as an empty tree entry. +fn splice(repo: &Repository, base: Option<&Tree>, path: &[&str], blob: Option) -> Result { + let (head, rest) = path.split_first().context("splice: empty path")?; + let mut tb = repo.treebuilder(base).context("treebuilder")?; + if rest.is_empty() { + match blob { + Some(oid) => { + tb.insert(*head, oid, 0o100644) + .context("inserting blob entry")?; + } + // Removing a never-committed path is a no-op, not an error (a note + // created and deleted between two syncs). + None => { + let _ = tb.remove(*head); + } + } + } else { + let sub = match base.and_then(|b| b.get_name(head)) { + Some(e) if e.kind() == Some(ObjectType::Tree) => { + Some(repo.find_tree(e.id()).context("loading subtree")?) + } + // Absent (a new directory) or a non-tree shadowing the name — + // build the subtree from scratch either way. + _ => None, + }; + let new_sub = splice(repo, sub.as_ref(), rest, blob)?; + if repo.find_tree(new_sub)?.len() == 0 { + let _ = tb.remove(*head); // the remove emptied this directory — prune it + } else { + tb.insert(*head, new_sub, 0o040000) + .context("inserting subtree entry")?; + } + } + tb.write().context("writing spliced tree") +} + +/// Origin's remote-tracking tip for `branch`, if the ref exists. libgit2 +/// updates it after a successful push/fetch, so it is "the newest commit we +/// know origin has" — without touching the network. +fn tracking_tip(repo: &Repository, branch: &str) -> Option { + repo.find_reference(&format!("refs/remotes/origin/{branch}")) + .ok()? + .peel_to_commit() + .ok() + .map(|c| c.id()) +} + +/// Whether origin is known to already have HEAD (local refs only, no network). +/// Errors read as "not current", so the caller falls through to the full +/// publish path where the real failure surfaces with context. +fn remote_current() -> Result { + let repo = Repository::open(REPO_DIR)?; + let head = repo.head()?.peel_to_commit()?.id(); + let branch = repo + .head()? + .shorthand() + .context("HEAD has no branch shorthand")? + .to_string(); + Ok(tracking_tip(&repo, &branch) == Some(head)) +} + +/// How a push attempt failed — this decides whether reconciling can help. +enum PushFailure { + /// The server processed the push but refused the ref update (arrives via + /// the `push_update_reference` callback — e.g. non-fast-forward): the + /// remote moved under us, and reconcile + replay is the right response. + Rejected(String), + /// Transport / TLS / auth / URL — the push never reached a ref decision, + /// so a reconcile (whose fetch needs the same transport) cannot help. + /// Surfaced directly; the 2026-07-13 on-device run burned a doomed + /// reconcile on an "unsupported URL protocol" because this wasn't split. + Other(anyhow::Error), +} + +impl PushFailure { + fn into_error(self) -> anyhow::Error { + match self { + Self::Rejected(msg) => anyhow::anyhow!("remote rejected ref: {msg}"), + Self::Other(e) => e, + } + } +} + /// One push attempt over HTTPS. Binds the PAT credential + the cert-verify -/// callback, and surfaces a server-side ref rejection (e.g. non-fast-forward) as -/// an error (it arrives via `push_update_reference`, not as a `push()` error). -fn try_push(repo: &Repository, refspec: &str) -> Result<()> { - let mut remote = repo.find_remote("origin")?; +/// callback, and separates a server-side ref rejection (reconcilable) from a +/// transport-level failure (not). +fn try_push(repo: &Repository, refspec: &str) -> Result<(), PushFailure> { + let mut remote = repo + .find_remote("origin") + .map_err(|e| PushFailure::Other(anyhow::Error::new(e).context("finding remote origin")))?; let rejection: Rc>> = Rc::new(RefCell::new(None)); let mut cbs = auth_callbacks(); @@ -353,27 +509,30 @@ fn try_push(repo: &Repository, refspec: &str) -> Result<()> { opts.remote_callbacks(cbs); remote .push(&[refspec], Some(&mut opts)) - .context("push transport")?; + .map_err(|e| PushFailure::Other(anyhow::Error::new(e).context("push transport")))?; if let Some(msg) = rejection.borrow().clone() { - bail!("remote rejected ref: {msg}"); + return Err(PushFailure::Rejected(msg)); } log::info!("push accepted by remote"); Ok(()) } -/// Fetch origin and mixed-reset the local branch onto it, so our just-made commit -/// can be replayed on the current tip. Only runs after a non-fast-forward push +/// Fetch origin and *soft*-reset the local branch onto it, so our changes can +/// be replayed on the current tip. Only runs after a non-fast-forward push /// rejection — i.e. the remote moved under us. /// -/// **MIXED**, deliberately not a force checkout: the note we're publishing lives -/// in the working tree, and a force checkout would clobber it. Mixed moves the -/// branch ref + index onto origin but leaves the working tree, so the note -/// survives and `stage_and_commit` replays it on top. For a single-writer -/// appliance this resolves last-writer-wins — a concurrent remote *edit* to the -/// same note loses to ours, and a remote-only *added* file the card doesn't have -/// is dropped by the replay's add --all. Both need a real merge (increment B) and -/// don't arise from this device's own use. +/// **SOFT**, deliberately: it moves only the branch ref. The previous Mixed +/// reset also rewrote the index — pure waste now that the splice commit never +/// reads the index, and on the real repo an index write is exactly the +/// racy-clean wall the splice exists to avoid. Neither flavor touches the +/// working tree, so the notes being published survive on the card and the +/// replay splices them onto the new tip. For a single-writer appliance this +/// resolves last-writer-wins: a concurrent remote *edit* to a note we're +/// publishing loses to ours, while a remote-only added/changed file is simply +/// carried forward — origin's tree is now the splice base, so the replay +/// keeps it (an improvement over the old `add --all` replay, which dropped +/// files the card didn't have). A real merge stays increment-B work. fn reconcile_onto_origin(repo: &Repository, branch: &str) -> Result<()> { let mut remote = repo.find_remote("origin")?; let mut fo = FetchOptions::new(); @@ -387,12 +546,12 @@ fn reconcile_onto_origin(repo: &Repository, branch: &str) -> Result<()> { .context("no FETCH_HEAD after fetch")?; let theirs = repo.reference_to_annotated_commit(&fetch_head)?; log::info!( - "reconcile: resetting local {branch} onto origin @ {} (mixed, keeps the note)", + "reconcile: resetting local {branch} onto origin @ {} (soft — ref move only, notes stay on the card)", short(theirs.id()) ); let their_obj = repo.find_object(theirs.id(), None)?; - repo.reset(&their_obj, git2::ResetType::Mixed, None) - .context("mixed reset onto origin")?; + repo.reset(&their_obj, git2::ResetType::Soft, None) + .context("soft reset onto origin")?; Ok(()) } diff --git a/firmware/src/main.rs b/firmware/src/main.rs index 95869b1..21e8abf 100644 --- a/firmware/src/main.rs +++ b/firmware/src/main.rs @@ -224,11 +224,21 @@ fn main() -> anyhow::Result<()> { // The outcome returns on `git_rx` and updates the snackbar // (see the idle branch below). The Save that preceded this // in the batch already persisted the buffer, so this is a - // pure git push. + // pure git publish of the recorded dirty paths — the + // outcome decides whether the snapshot is forgotten + // (publish_succeeded) or retried (publish_failed). #[cfg(feature = "git")] - match git_tx.send(firmware::git_sync::PublishRequest) { - Ok(()) => ed.set_notice("syncing..."), - Err(_) => ed.set_notice("sync: git thread down"), + { + let paths = storage.take_dirty(); + match git_tx.send(firmware::git_sync::PublishRequest { paths }) { + Ok(()) => ed.set_notice("syncing..."), + Err(_) => { + // Thread gone — nothing will report back, so + // return the snapshot to pending ourselves. + storage.publish_failed(); + ed.set_notice("sync: git thread down"); + } + } } #[cfg(not(feature = "git"))] log::info!(":sync — saved; light build (no `git` feature) — push skipped"); @@ -259,6 +269,13 @@ fn main() -> anyhow::Result<()> { #[cfg(feature = "git")] if let Ok(outcome) = git_rx.try_recv() { use firmware::git_sync::PublishOutcome::*; + // Settle the dirty snapshot this publish took: confirmed + // published (or up to date) → forget it; failed → back to + // pending so the next :sync retries the same paths. + match &outcome { + Pushed(_) | UpToDate => storage.publish_succeeded(), + Failed(_) => storage.publish_failed(), + } ed.set_notice(match outcome { Pushed(oid) => format!("synced {oid}"), UpToDate => "up to date".to_string(), @@ -410,7 +427,15 @@ fn main() -> anyhow::Result<()> { /// `main`): the note is the whole point of the appliance, so we refuse to run /// in a state where the next save could destroy it. fn boot_storage(epd: &mut Epd) -> (Storage, String) { - let storage = match Storage::mount() { + // A git build shares this mount with the git thread, and libgit2 keeps the + // pack + idx descriptors open across a publish — that overruns the + // editor's tight 4-FD budget, so mount with the 16-FD one (persistence.rs, + // MAX_FILES_GIT). The light build keeps the editor's own budget. + #[cfg(feature = "git")] + let mounted = Storage::mount_for_git(); + #[cfg(not(feature = "git"))] + let mounted = Storage::mount(); + let storage = match mounted { Ok(s) => s, Err(e) => boot_halt(epd, "SD card not ready", &format!("{e:#}")), }; diff --git a/firmware/src/persistence.rs b/firmware/src/persistence.rs index ff0255b..fcc26a5 100644 --- a/firmware/src/persistence.rs +++ b/firmware/src/persistence.rs @@ -34,6 +34,8 @@ //! sits in the tmp. [`Storage::recover`] closes the loop at boot — see its docs //! for the exact case analysis, which is subtler than "promote the tmp." +use std::cell::RefCell; +use std::collections::BTreeSet; use std::fs; use std::io::Write as _; use std::mem::MaybeUninit; @@ -79,6 +81,14 @@ pub const MAX_FILE_BYTES: u64 = 256 * 1024; /// The C mount point (`/sd\0`) for the esp-idf FFI calls. const MOUNT_C: &std::ffi::CStr = c"/sd"; +/// Dirty-path journal — one repo-relative path per line, mirroring the in-RAM +/// dirty set (see [`Storage::take_dirty`]). At the card root, *outside* +/// `/sd/repo`, so it can never itself be committed. Without it a power pull +/// would strand every file saved-but-not-yet-published in that session: the +/// splice commit only visits recorded paths (nothing walks the tree anymore), +/// so an unrecorded change would never reach the remote. +const DIRTY_JOURNAL: &str = "/sd/.typoena-dirty"; + /// VFS open-file budget for the editor path: it opens only a note and its /// `*.tmp`, so a tight budget keeps FatFS's per-file buffers off the heap. const MAX_FILES_EDITOR: i32 = 4; @@ -95,6 +105,23 @@ const MAX_FILES_GIT: i32 = 16; /// lock serialises the two, so no extra mutex is needed here. pub struct Storage { card: *mut sys::sdmmc_card_t, + /// Repo-relative paths saved or `:delete`d since the last confirmed + /// publish — the editor-side half of the O(depth) splice commit + /// (`git_sync::stage_and_commit` visits exactly these paths and nothing + /// else). Mirrored to [`DIRTY_JOURNAL`] whenever it changes, so the record + /// survives a power pull. `RefCell` because recording happens inside + /// `&self` save/delete calls; `Storage` already lives on one task only. + dirty: RefCell, +} + +/// The two halves of the dirty record: `pending` accumulates between syncs; +/// `take_dirty` moves it to `in_flight` for the duration of a publish so a +/// failure can put it back (and a save landing *during* the publish re-enters +/// `pending`, riding the next one). The journal always carries the union. +#[derive(Default)] +struct Dirty { + pending: BTreeSet, + in_flight: BTreeSet, } /// What [`Storage::recover`] did with a leftover `*.tmp` at boot. @@ -223,7 +250,10 @@ impl Storage { } esp!(rc).context("esp_vfs_fat_sdspi_mount (card present? inserted? FAT-formatted?)")?; - let storage = Storage { card }; + let storage = Storage { + card, + dirty: RefCell::new(Dirty::default()), + }; let (max_khz, real_khz) = storage.negotiated_khz(); log::info!("SD mounted at {MOUNT} — max {max_khz} kHz, negotiated {real_khz} kHz"); @@ -238,6 +268,13 @@ impl Storage { newest complete copy)" ), } + let carried = storage.load_dirty_journal(); + if carried > 0 { + log::info!( + "dirty journal: {carried} unpublished path(s) carried over from a previous \ + session — the next :sync will commit them" + ); + } Ok(storage) } @@ -314,6 +351,16 @@ impl Storage { /// buffers is deferred to the v0.9 crash-safety work — the atomic swap here /// already protects each individual save. pub fn save_path(&self, path: &str, contents: &str) -> Result<()> { + // Record BEFORE writing: a crash in between leaves an over-approximate + // journal (the splice of an unchanged path is a no-op), whereas the + // reverse order could leave a changed file no record ever points at. + self.record_dirty(path); + Self::atomic_write(path, contents) + } + + /// The atomic write primitive behind [`Storage::save_path`] and the dirty + /// journal: write `{path}.tmp`, fsync, unlink the target, rename over it. + fn atomic_write(path: &str, contents: &str) -> Result<()> { let tmp = format!("{path}.tmp"); { let mut f = fs::File::create(&tmp) @@ -350,6 +397,9 @@ impl Storage { /// file half-present after a delete. For a Tracked file this leaves the /// working copy short one file; the next publish's `add --all` stages it. pub fn delete_path(&self, path: &str) -> Result<()> { + // Same record-first rule as `save_path`: the splice treats a recorded + // path with no file behind it as "remove from the tree". + self.record_dirty(path); let _ = fs::remove_file(format!("{path}.tmp")); match fs::remove_file(path) { Ok(()) => Ok(()), @@ -358,6 +408,88 @@ impl Storage { } } + /// Note a working-copy file as (possibly) differing from HEAD. Paths + /// outside `/sd/repo` (`/sd/local`, `/sd/ca.pem`, the journal itself) are + /// not git's business and are skipped. The journal is rewritten only when + /// the set actually grows, so re-saving the same note between syncs costs + /// no extra card I/O. + fn record_dirty(&self, abs_path: &str) { + let Some(rel) = abs_path + .strip_prefix(REPO_DIR) + .and_then(|r| r.strip_prefix('/')) + else { + return; + }; + if rel.is_empty() { + return; + } + let grew = self.dirty.borrow_mut().pending.insert(rel.to_string()); + if grew { + self.persist_dirty(); + } + } + + /// Snapshot the dirty paths for a publish (repo-relative). The snapshot + /// moves to `in_flight` — the journal keeps carrying it — until the UI + /// task reports the outcome: [`Storage::publish_succeeded`] forgets it, + /// [`Storage::publish_failed`] returns it to pending for the next `:sync`. + pub fn take_dirty(&self) -> BTreeSet { + let mut d = self.dirty.borrow_mut(); + let taken = std::mem::take(&mut d.pending); + d.in_flight.extend(taken.iter().cloned()); + taken + } + + /// The publish that took the last snapshot committed (or confirmed + /// up-to-date): drop its paths and shrink the journal. Anything saved + /// while it ran is still in `pending` and rides the next sync. + pub fn publish_succeeded(&self) { + self.dirty.borrow_mut().in_flight.clear(); + self.persist_dirty(); + } + + /// The publish failed: return its snapshot to pending so the next `:sync` + /// retries it (the splice is idempotent, so a retry of an already-clean + /// path is free). The journal already carries these paths — no rewrite. + pub fn publish_failed(&self) { + let mut d = self.dirty.borrow_mut(); + let inflight = std::mem::take(&mut d.in_flight); + d.pending.extend(inflight); + } + + /// Mirror `pending ∪ in_flight` to [`DIRTY_JOURNAL`], atomically. + /// Best-effort: a failed journal write must not fail the save that + /// triggered it — the set stays correct in RAM and the journal heals on + /// the next change. + fn persist_dirty(&self) { + let contents = { + let d = self.dirty.borrow(); + let mut out = String::new(); + for p in d.pending.union(&d.in_flight) { + out.push_str(p); + out.push('\n'); + } + out + }; + if let Err(e) = Self::atomic_write(DIRTY_JOURNAL, &contents) { + log::warn!("dirty journal write FAILED ({e:#}); set kept in RAM only"); + } + } + + /// Seed the dirty set from the journal at mount — the paths a previous + /// session saved but never got confirmed as published (power pull, failed + /// sync, or simply no `:sync` before shutdown). Returns how many. + fn load_dirty_journal(&self) -> usize { + let Ok(text) = fs::read_to_string(DIRTY_JOURNAL) else { + return 0; // no journal yet — nothing carried over + }; + let mut d = self.dirty.borrow_mut(); + for line in text.lines().map(str::trim).filter(|l| !l.is_empty()) { + d.pending.insert(line.to_string()); + } + d.pending.len() + } + /// Reconcile a leftover `notes.md.tmp` at boot. The save sequence is /// write-tmp → fsync → unlink-target → rename, so a lingering tmp means the /// last save was interrupted. Which way to recover depends on whether the