diff --git a/editor/src/lib.rs b/editor/src/lib.rs index 4603e53..af1939a 100644 --- a/editor/src/lib.rs +++ b/editor/src/lib.rs @@ -753,11 +753,22 @@ pub struct Editor { /// Host-effect queue, drained by [`take_effects`](Self::take_effects) after a /// key batch. See [`Effect`]. requests: Vec, - /// Every openable file, as absolute paths, fed by the host at boot via - /// [`set_file_list`](Self::set_file_list) (a recursive walk of `/sd/repo` - /// and `/sd/local`). The palette fuzzy-filters this once the query reaches - /// [`PALETTE_MIN_QUERY`] chars; empty until the host feeds it. - files: Vec, + /// Every openable file, as absolute paths, fed by the host at boot (a + /// recursive walk of `/sd/repo` and `/sd/local`). The palette fuzzy-filters + /// this once the query reaches [`PALETTE_MIN_QUERY`] chars; empty until the + /// host feeds it. + /// + /// **Interned**: one newline-joined blob plus byte spans, not a + /// `Vec`. On the device 1099 paths as individual `String`s cost + /// 182 KB of *internal* DRAM (small allocs stay under the 16 KB + /// PSRAM-malloc threshold, and per-alloc overhead dwarfs the ~50-byte + /// payloads) — which starved the SD DMA pool during the first on-device + /// pull (2026-07-14). A single large blob lands in PSRAM; the spans are + /// one small `Vec`. Access via [`file_at`](Self::file_at). + file_blob: String, + /// Byte ranges of each path in [`file_blob`](Self::file_blob), sorted by + /// the paths they point at (the palette's stable base order). + file_spans: Vec<(u32, u32)>, /// Recently-opened files, most-recent-first (an MRU), deduped and bounded to /// [`MRU_MAX`]. Every `:e`/palette open pushes to the front /// ([`note_recent`](Self::note_recent)); it orders the palette when the query @@ -867,7 +878,8 @@ impl Editor { dirty: false, parked: Vec::new(), requests: Vec::new(), - files: Vec::new(), + file_blob: String::new(), + file_spans: Vec::new(), recent: Vec::new(), palette_query: String::new(), palette_sel: 0, @@ -1864,31 +1876,76 @@ impl Editor { } } - /// Insert `path` into the palette's file list, keeping it sorted and unique - /// (matches [`set_file_list`](Self::set_file_list)'s invariant). Used by - /// `:enew` so a just-created file is findable without a disk re-enumeration. + /// The `i`-th file path in the palette's sorted base order (a slice into + /// [`file_blob`](Self::file_blob)). + fn file_at(&self, i: usize) -> &str { + let (s, e) = self.file_spans[i]; + &self.file_blob[s as usize..e as usize] + } + + /// How many files the palette knows about. + fn file_count(&self) -> usize { + self.file_spans.len() + } + + /// Insert `path` into the palette's file list, keeping the spans sorted and + /// unique (matches [`set_file_list_joined`](Self::set_file_list_joined)'s + /// invariant). Used by `:enew` so a just-created file is findable without a + /// disk re-enumeration. Appends to the blob; a `String` realloc only moves + /// bytes, the spans are indices and stay valid. fn add_to_file_list(&mut self, path: &str) { - if let Err(i) = self.files.binary_search(&path.to_string()) { - self.files.insert(i, path.to_string()); + match self + .file_spans + .binary_search_by(|&(s, e)| self.file_blob[s as usize..e as usize].cmp(path)) + { + Ok(_) => {} + Err(i) => { + let start = self.file_blob.len() as u32; + self.file_blob.push_str(path); + self.file_spans.insert(i, (start, start + path.len() as u32)); + } } } - /// Drop `path` from the palette's file list (used by `:delete`). + /// Drop `path` from the palette's file list (used by `:delete`). Only the + /// span goes; its bytes stay in the blob as dead weight until the next + /// host re-walk replaces the whole thing — a few dozen bytes at most. fn remove_from_file_list(&mut self, path: &str) { - self.files.retain(|f| f != path); + let blob = &self.file_blob; + self.file_spans + .retain(|&(s, e)| &blob[s as usize..e as usize] != path); } // --- File palette (Ctrl-P) --------------------------------------------- - /// Feed the palette its file list: every openable file as an absolute path, - /// enumerated by the host from `/sd/repo` and `/sd/local` (at boot for v0.5). - /// Sorted + deduped for a stable base order; the MRU floats recents above it. - /// The palette is a pure view over this — nothing is read from disk until a - /// file is actually opened. - pub fn set_file_list(&mut self, mut files: Vec) { - files.sort(); - files.dedup(); - self.files = files; + /// Feed the palette its file list as **one newline-joined blob** of + /// absolute paths, enumerated by the host from `/sd/repo` and `/sd/local`. + /// This is the device's entry point: a single large `String` lands in + /// PSRAM (allocations ≥ 16 KB cross the SPIRAM-malloc threshold), where + /// the same list as 1099 individual `String`s measured 182 KB of internal + /// DRAM. Spans are sorted + deduped for a stable base order; the MRU + /// floats recents above it. The palette is a pure view over this — nothing + /// is read from disk until a file is actually opened. + pub fn set_file_list_joined(&mut self, blob: String) { + let mut spans: Vec<(u32, u32)> = Vec::new(); + let mut start = 0u32; + for line in blob.split('\n') { + let end = start + line.len() as u32; + if !line.is_empty() { + spans.push((start, end)); + } + start = end + 1; // past the '\n' + } + spans.sort_by(|&(a, b), &(c, d)| blob[a as usize..b as usize].cmp(&blob[c as usize..d as usize])); + spans.dedup_by(|&mut (a, b), &mut (c, d)| blob[a as usize..b as usize] == blob[c as usize..d as usize]); + self.file_blob = blob; + self.file_spans = spans; + } + + /// [`set_file_list_joined`](Self::set_file_list_joined) from a `Vec` — + /// convenience for hosts/tests that already hold separate strings. + pub fn set_file_list(&mut self, files: Vec) { + self.set_file_list_joined(files.join("\n")); } /// Push `path` to the front of the recent-files MRU (dropping any earlier @@ -2042,7 +2099,7 @@ impl Editor { let idx = self.palette_matches().get(self.palette_sel).copied(); self.close_palette(); let Some(idx) = idx else { return }; - let (path, scope) = resolve_path(&self.files[idx], self.scope); + let (path, scope) = resolve_path(self.file_at(idx), self.scope); self.open_path(path, scope); } @@ -2056,14 +2113,14 @@ impl Editor { /// through unranked, but the MRU keeps quick-switch (`Cmd-P`, `Enter`) one /// keystroke away. Two typed chars reveal the full list. fn palette_matches(&self) -> Vec { - let mut order: Vec = Vec::with_capacity(self.files.len()); + let mut order: Vec = Vec::with_capacity(self.file_count()); for r in &self.recent { - if let Some(i) = self.files.iter().position(|f| f == r) { + if let Some(i) = (0..self.file_count()).find(|&i| self.file_at(i) == r) { order.push(i); } } if self.palette_query.chars().count() >= PALETTE_MIN_QUERY { - for i in 0..self.files.len() { + for i in 0..self.file_count() { if !order.contains(&i) { order.push(i); } @@ -2075,7 +2132,7 @@ impl Editor { let mut scored: Vec<(usize, i32)> = order .into_iter() .filter_map(|i| { - fuzzy_score(&self.palette_query, palette_label(&self.files[i])).map(|s| (i, s)) + fuzzy_score(&self.palette_query, palette_label(self.file_at(i))).map(|s| (i, s)) }) .collect(); // Stable sort by descending score — ties keep their MRU/base position. @@ -3707,7 +3764,7 @@ impl Editor { } } else if command_mode { "(no command)" - } else if self.files.is_empty() { + } else if self.file_spans.is_empty() { "(no files on card)" } else if self.palette_query.chars().count() < PALETTE_MIN_QUERY { // No recents yet and the query is below the search threshold — @@ -3730,7 +3787,7 @@ impl Editor { } else if command_mode { self.command_label(PALETTE_CMDS[idx]) } else { - palette_label(&self.files[idx]).chars().take(max_chars).collect() + palette_label(self.file_at(idx)).chars().take(max_chars).collect() }; if row == sel { // Reverse video: black fill across the column, white glyphs. @@ -5231,7 +5288,7 @@ mod tests { fn enew_adds_the_new_file_to_the_palette_list() { let mut e = palette_editor(&["/sd/repo/notes.md", "/sd/repo/todo.md"]); ex(&mut e, "enew draft.md"); - assert!(e.files.contains(&"/sd/repo/draft.md".to_string())); + assert!(files_vec(&e).contains(&"/sd/repo/draft.md".to_string())); // and it is findable in the palette without a disk re-enumeration e.handle(Key::Palette); for c in "draft".chars() { @@ -5306,7 +5363,7 @@ mod tests { let mut e = palette_editor(&["/sd/repo/notes.md", "/sd/repo/todo.md"]); ex(&mut e, "delete"); // notes.md is active e.take_effects(); - assert!(!e.files.contains(&"/sd/repo/notes.md".to_string())); + assert!(!files_vec(&e).contains(&"/sd/repo/notes.md".to_string())); e.handle(Key::Palette); for c in "md".chars() { e.handle(Key::Char(c)); // reach the search threshold @@ -5346,7 +5403,12 @@ mod tests { /// The palette's current result as display labels, in ranked order. fn palette_labels(e: &Editor) -> Vec<&str> { - e.palette_matches().iter().map(|&i| palette_label(&e.files[i])).collect() + e.palette_matches().iter().map(|&i| palette_label(e.file_at(i))).collect() + } + + /// The interned file list back as owned strings, in base (sorted) order. + fn files_vec(e: &Editor) -> Vec { + (0..e.file_count()).map(|i| e.file_at(i).to_string()).collect() } #[test] @@ -5374,7 +5436,7 @@ mod tests { "/sd/repo/a.md".into(), "/sd/repo/b.md".into(), ]); - assert_eq!(e.files, vec!["/sd/repo/a.md", "/sd/repo/b.md"]); + assert_eq!(files_vec(&e), vec!["/sd/repo/a.md", "/sd/repo/b.md"]); } #[test] @@ -6517,6 +6579,21 @@ mod tests { assert_eq!(e.caret, 10); // boot posture: caret on the last char } + #[test] + fn joined_file_list_sorts_dedups_and_survives_blank_lines() { + let mut e = Editor::new(); + e.set_file_list_joined("/sd/repo/b.md\n\n/sd/repo/a.md\n/sd/repo/b.md\n".into()); + assert_eq!(files_vec(&e), vec!["/sd/repo/a.md", "/sd/repo/b.md"]); + e.add_to_file_list("/sd/repo/ab.md"); // lands between, sorted + e.add_to_file_list("/sd/repo/a.md"); // already known — no dup + assert_eq!( + files_vec(&e), + vec!["/sd/repo/a.md", "/sd/repo/ab.md", "/sd/repo/b.md"] + ); + e.remove_from_file_list("/sd/repo/b.md"); + assert_eq!(files_vec(&e), vec!["/sd/repo/a.md", "/sd/repo/ab.md"]); + } + #[test] fn drop_clean_parked_keeps_only_dirty_buffers() { let mut e = over("one"); // active: notes.md, clean diff --git a/firmware/src/main.rs b/firmware/src/main.rs index 4cf84fc..176b79d 100644 --- a/firmware/src/main.rs +++ b/firmware/src/main.rs @@ -87,7 +87,7 @@ fn main() -> anyhow::Result<()> { // must not hold up the first editor frame. The list lands on `walk_rx` and // the idle branch of the main loop feeds it to the editor; until then the // palette shows recents only. A pull re-feeds it the same way. - let (walk_tx, walk_rx) = std::sync::mpsc::channel::>(); + let (walk_tx, walk_rx) = std::sync::mpsc::channel::(); spawn_file_walk(walk_tx.clone()); // Bring up the USB keyboard in the background; keys arrive via next_key(). @@ -276,6 +276,10 @@ fn main() -> anyhow::Result<()> { { use firmware::git_sync::GitRequest; if storage.has_dirty() { + // Log it too — on the 2026-07-14 run this gate + // firing looked like a silent no-op in the + // serial log. + log::info!(":gl refused — dirty journal non-empty; :sync first"); ed.set_notice("pull: unsynced changes - :sync first"); } else { match git_tx.send(GitRequest::Pull) { @@ -376,7 +380,7 @@ fn main() -> anyhow::Result<()> { // ~630 ms panel drive. Caret visibility is passed through // unchanged so this can't reveal a debounced Insert caret early. if let Ok(files) = walk_rx.try_recv() { - ed.set_file_list(files); + ed.set_file_list_joined(files); ed.draw_into(&mut back, cursor_shown); if changed_rows(shown.bytes(), back.bytes()).is_some() { if let Err(e) = epd.display_frame_partial_window(back.bytes(), 0, epd::HEIGHT) { @@ -648,20 +652,27 @@ fn delete_buffer(storage: &Storage, ed: &mut Editor, path: String, scope: Scope) } /// Enumerate the palette's openable files: the regular files under `/sd/repo` -/// and `/sd/local`, recursively, as absolute paths. Skips dot entries at every -/// level (so `.git` and its thousands of object files, `.typoena.toml`, and the -/// like never show or get walked). Best-effort: an unreadable directory (e.g. -/// no `/sd/local` yet) contributes nothing rather than failing. The editor -/// sorts and dedupes. Runs on the `walk` thread (`spawn_file_walk`), so the -/// walk time is logged — on a big repo the FAT directory IO is the cost to -/// watch (~4 ms/file over SPI). -fn enumerate_files() -> Vec { +/// and `/sd/local`, recursively, as absolute paths — **one newline-joined +/// blob**, not a `Vec`. 1099 paths as individual small `String`s +/// measured 182 KB of *internal* DRAM resident (each stays under the 16 KB +/// SPIRAM-malloc threshold, plus per-alloc overhead), which starved the SD DMA +/// pool during the first on-device pull (2026-07-14). The blob is seeded past +/// the threshold so it and its growth reallocs land in PSRAM. Skips dot +/// entries at every level (so `.git` and its thousands of object files never +/// get walked). Best-effort: an unreadable directory (e.g. no `/sd/local` +/// yet) contributes nothing rather than failing. The editor sorts and dedupes +/// span-side. Runs on the `walk` thread (`spawn_file_walk`); on a big repo +/// the FAT directory IO is the cost to watch (~4 ms/file over SPI). +fn enumerate_files() -> String { let start = std::time::Instant::now(); - let mut out = Vec::new(); + // 64 KB seed: comfortably past the 16 KB SPIRAM threshold and roomy enough + // that a ~1100-file tree never reallocs. + let mut out = String::with_capacity(64 * 1024); + let mut count = 0usize; for dir in [REPO_DIR, LOCAL_DIR] { - walk_files(std::path::Path::new(dir), 0, &mut out); + walk_files(std::path::Path::new(dir), 0, &mut out, &mut count); } - log::info!("file walk: {} files in {}ms", out.len(), start.elapsed().as_millis()); + log::info!("file walk: {count} files in {}ms", start.elapsed().as_millis()); out } @@ -671,11 +682,9 @@ fn enumerate_files() -> Vec { /// seconds on a big tree and the palette is not mandatory for typing. The /// walk is pure directory reads, serialized against the editor's and the git /// thread's SD traffic by the FatFS volume lock. Bracketed with internal-DRAM -/// readings: each path is a small String, kept internal by the SPIRAM malloc -/// threshold (16 KB), so the list competes with Wi-Fi/TLS for DRAM — the -/// logged number decides whether interning the paths into one shared buffer -/// (a single >16 KB alloc, which lands in PSRAM) is worth the refactor. -fn spawn_file_walk(tx: std::sync::mpsc::Sender>) { +/// readings to confirm the interned blob keeps the list out of internal +/// (pre-interning: 182 KB resident; expected now: ~0, the spans only). +fn spawn_file_walk(tx: std::sync::mpsc::Sender) { // Explicit stack: the default pthread stack (4 KB) is tight for 8 levels // of readdir recursion plus FatFS underneath. let spawned = std::thread::Builder::new() @@ -686,8 +695,9 @@ fn spawn_file_walk(tx: std::sync::mpsc::Sender>) { let files = enumerate_files(); let dram_after = internal_free_heap(); log::info!( - "file list: internal heap {dram_before} -> {dram_after} ({} KB consumed)", - dram_before.saturating_sub(dram_after) / 1024 + "file list: internal heap {dram_before} -> {dram_after} ({} KB consumed), blob {} KB", + dram_before.saturating_sub(dram_after) / 1024, + files.len() / 1024 ); let _ = tx.send(files); // receiver gone = shutting down; nothing to do }); @@ -705,7 +715,7 @@ const WALK_MAX_DEPTH: usize = 8; /// recursing (the `remove_dir_recursive` pattern in `git_sync`), so only one /// FatFS directory handle is open at a time regardless of depth — relevant on /// the FD-bounded SD mount. -fn walk_files(dir: &std::path::Path, depth: usize, out: &mut Vec) { +fn walk_files(dir: &std::path::Path, depth: usize, out: &mut String, count: &mut usize) { if depth > WALK_MAX_DEPTH { log::warn!("file walk: {} exceeds depth {WALK_MAX_DEPTH}, skipped", dir.display()); return; @@ -750,10 +760,12 @@ fn walk_files(dir: &std::path::Path, depth: usize, out: &mut Vec) { }; if is_file { if let Some(p) = path.to_str() { - out.push(p.to_string()); + out.push_str(p); + out.push('\n'); + *count += 1; } } else if is_dir { - walk_files(&path, depth + 1, out); + walk_files(&path, depth + 1, out, count); } } }