#48 Stream oversized blobs to raw/ instead of buffering (#66)

src/build.rs+70 −34
diff --git a/src/build.rs b/src/build.rsindex 90e6651..e3ba54a 100644--- a/src/build.rs+++ b/src/build.rs@@ -196,8 +196,13 @@ pub fn find_readme(repo: &Path, branch: &str) -> Option<(String, gitx::Blob)> {196196         ".forge/wiki/README",197197         ".forge/wiki/readme",198198     ] {199-        if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) {200-            return Some((candidate.to_string(), blob));199+        let fits = gitx::blob_size(repo, branch, Path::new(candidate))200+            .map(|size| size <= README_MAX_BYTES)201+            .unwrap_or(false);202+        if fits {203+            if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) {204+                return Some((candidate.to_string(), blob));205+            }201206         }202207     }203208     None@@ -209,6 +214,17 @@ pub fn is_placeholder(bytes: &[u8]) -> bool {209214     bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0)210215 }211216 217+/// Blobs above this size are streamed straight to `raw/` instead of218+/// being held in memory (f0rge #66). Reading every blob fully used to219+/// abort the process in the allocator on a few-hundred-MB file; the220+/// page for such a blob is a placeholder anyway, so only the raw copy221+/// needs the bytes.222+pub const MAX_INLINE_BLOB: u64 = 8 * 1024 * 1024;223+224+/// READMEs above this size are ignored (next candidate in the chain is225+/// tried). Prevents a huge README from bloating every page's preview.226+pub const README_MAX_BYTES: u64 = 1024 * 1024;227+212228 /// Bytes sniffed for the language-math binary check (first 8KiB).213229 pub const BINARY_SNIFF_BYTES: usize = 8 * 1024;214230 @@ -955,16 +971,27 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu955971         wiki::page_count(repo) > 0,956972     );957973 958-    // Raw files first: byte-identical copies.974+    // Raw files first: byte-identical copies. Blobs over MAX_INLINE_BLOB975+    // are streamed to disk and never enter the in-memory map; their file976+    // pages become placeholders (f0rge #66).959977     let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();978+    let mut oversize: BTreeMap<String, u64> = BTreeMap::new();960979     for f in &files {961-        let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;962980         let dest = out.join("raw").join(f);963981         if let Some(parent) = dest.parent() {964982             std::fs::create_dir_all(parent).map_err(|e| format!("hu13: create raw parent: {e}"))?;965983         }966-        std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?;967-        blobs.insert(f.clone(), blob.bytes);984+        let size =985+            gitx::blob_size(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;986+        if size > MAX_INLINE_BLOB {987+            gitx::show_to_file(repo, &branch, Path::new(f), &dest)988+                .map_err(|e| format!("hu13: {e}"))?;989+            oversize.insert(f.clone(), size);990+        } else {991+            let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;992+            std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?;993+            blobs.insert(f.clone(), blob.bytes);994+        }968995     }969996 970997     // Per-path pages (m6 code-gitea, f0rge #33): Gitea-style file@@ -1038,41 +1065,50 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu10381065     }10391066 10401067     for f in &files {1041-        let bytes = blobs.get(f).cloned().unwrap_or_default();10421068         let mut body = breadcrumb(f);10431069         body.push_str(&format!("<h1>{}</h1>\n", esc(f)));1044-        if is_placeholder(&bytes) {1045-            body.push_str(&file_meta_bar(f, bytes.len() as u64, None));1046-            let kind = if bytes.len() >= PREVIEW_MAX_BYTES {1047-                format!("too large to preview ({} bytes)", bytes.len())1048-            } else {1049-                "binary file".to_string()1050-            };1070+        if let Some(&size) = oversize.get(f) {1071+            body.push_str(&file_meta_bar(f, size, None));10511072             body.push_str(&format!(1052-                "<p class=\"notice\">placeholder: {kind} — preview omitted. \1073+                "<p class=\"notice\">placeholder: too large to preview ({size} bytes) — \10531074                  <a href=\"/raw/{}\">download the raw file</a> instead.</p>\n",10541075                 href_esc(f)10551076             ));1056-        } else if is_markdown(f) {1057-            let display = String::from_utf8_lossy(&bytes).into_owned();1058-            body.push_str(&file_meta_bar(1059-                f,1060-                bytes.len() as u64,1061-                Some(display.lines().count()),1062-            ));1063-            body.push_str(&markdown_to_html(&display));1064-            body.push_str(&format!(1065-                "<p class=\"muted\"><a href=\"/raw/{}\">view raw</a></p>\n",1066-                href_esc(f)1067-            ));10681077         } else {1069-            let display = String::from_utf8_lossy(&bytes).into_owned();1070-            body.push_str(&file_meta_bar(1071-                f,1072-                bytes.len() as u64,1073-                Some(display.lines().count()),1074-            ));1075-            body.push_str(&render_text_pre(&display));1078+            let bytes = blobs.get(f).cloned().unwrap_or_default();1079+            if is_placeholder(&bytes) {1080+                body.push_str(&file_meta_bar(f, bytes.len() as u64, None));1081+                let kind = if bytes.len() >= PREVIEW_MAX_BYTES {1082+                    format!("too large to preview ({} bytes)", bytes.len())1083+                } else {1084+                    "binary file".to_string()1085+                };1086+                body.push_str(&format!(1087+                    "<p class=\"notice\">placeholder: {kind} — preview omitted. \1088+                 <a href=\"/raw/{}\">download the raw file</a> instead.</p>\n",1089+                    href_esc(f)1090+                ));1091+            } else if is_markdown(f) {1092+                let display = String::from_utf8_lossy(&bytes).into_owned();1093+                body.push_str(&file_meta_bar(1094+                    f,1095+                    bytes.len() as u64,1096+                    Some(display.lines().count()),1097+                ));1098+                body.push_str(&markdown_to_html(&display));1099+                body.push_str(&format!(1100+                    "<p class=\"muted\"><a href=\"/raw/{}\">view raw</a></p>\n",1101+                    href_esc(f)1102+                ));1103+            } else {1104+                let display = String::from_utf8_lossy(&bytes).into_owned();1105+                body.push_str(&file_meta_bar(1106+                    f,1107+                    bytes.len() as u64,1108+                    Some(display.lines().count()),1109+                ));1110+                body.push_str(&render_text_pre(&display));1111+            }10761112         }10771113         let html = crate::layout::page_site_with_provenance(10781114             &format!("{f} — {name}"),
src/gitx.rs+123 −1
diff --git a/src/gitx.rs b/src/gitx.rsindex ac68f8f..8d09364 100644--- a/src/gitx.rs+++ b/src/gitx.rs@@ -7,7 +7,7 @@77 #![allow(dead_code)]88 99 use std::ffi::{OsStr, OsString};10-use std::io::Read;10+use std::io::{Read, Write};1111 use std::path::{Path, PathBuf};1212 use std::process::{Command, Stdio};1313 use std::sync::mpsc;@@ -669,6 +669,128 @@ pub fn show(repo: &Path, branch: &str, path: &Path) -> Result<Blob, GitError> {669669     })670670 }671671 672+/// Size in bytes of `branch:path` without reading it (`git cat-file -s`).673+///674+/// Callers use this to avoid loading very large blobs into memory675+/// (f0rge #66): a 400 MB blob used to abort the process in the676+/// allocator.677+pub fn blob_size(repo: &Path, branch: &str, path: &Path) -> Result<u64, GitError> {678+    let mut spec = OsString::from(branch);679+    spec.push(":");680+    spec.push(path.as_os_str());681+    let a_cat: &OsStr = OsStr::new("cat-file");682+    let a_size: &OsStr = OsStr::new("-s");683+    let args: Vec<&OsStr> = vec![a_cat, a_size, spec.as_os_str()];684+    let out = run_git(repo, &args)?;685+    if !out.status_ok {686+        let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();687+        return Err(GitError::NotFound(format!(688+            "`{branch}:{}` not found: {stderr}",689+            path.to_string_lossy()690+        )));691+    }692+    String::from_utf8_lossy(&out.stdout)693+        .trim()694+        .parse::<u64>()695+        .map_err(|e| GitError::Failed {696+            cmd: cmd_string(repo, &args),697+            status: "parse".to_string(),698+            stderr: format!("cat-file -s: {e}"),699+        })700+}701+702+/// Stream `branch:path` to `dest` without buffering it in the process.703+///704+/// Used for blobs above the in-memory cap (f0rge #66). The destination705+/// is removed again if git fails or the timeout fires, so a partial706+/// file never survives a failed build.707+pub fn show_to_file(repo: &Path, branch: &str, path: &Path, dest: &Path) -> Result<(), GitError> {708+    let mut spec = OsString::from(branch);709+    spec.push(":");710+    spec.push(path.as_os_str());711+    let a_show: &OsStr = OsStr::new("show");712+    let args: Vec<&OsStr> = vec![a_show, spec.as_os_str()];713+714+    let mut cmd = Command::new("git");715+    cmd.arg("-C")716+        .arg(repo)717+        .args(&args)718+        .stdout(Stdio::piped())719+        .stderr(Stdio::piped());720+    let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?;721+    let stdout = child.stdout.take();722+    let stderr = child.stderr.take();723+724+    let (tx_err, rx_err) = mpsc::channel::<Vec<u8>>();725+    if let Some(mut err) = stderr {726+        thread::spawn(move || {727+            let mut buf = Vec::new();728+            let _ = err.read_to_end(&mut buf);729+            let _ = tx_err.send(buf);730+        });731+    } else {732+        let _ = tx_err.send(Vec::new());733+    }734+735+    let dest_owned = dest.to_path_buf();736+    let (tx_copy, rx_copy) = mpsc::channel::<std::io::Result<u64>>();737+    if let Some(mut out) = stdout {738+        thread::spawn(move || {739+            let file = match std::fs::File::create(&dest_owned) {740+                Ok(f) => f,741+                Err(e) => {742+                    let _ = tx_copy.send(Err(e));743+                    return;744+                }745+            };746+            let mut w = std::io::BufWriter::new(file);747+            let res = std::io::copy(&mut out, &mut w).and_then(|n| w.flush().map(|()| n));748+            let _ = tx_copy.send(res);749+        });750+    } else {751+        let _ = tx_copy.send(Err(std::io::Error::other("git show produced no stdout")));752+    }753+754+    let start = Instant::now();755+    let status = loop {756+        match child.try_wait() {757+            Err(e) => {758+                let _ = std::fs::remove_file(dest);759+                return Err(GitError::Failed {760+                    cmd: cmd_string(repo, &args),761+                    status: "wait-error".to_string(),762+                    stderr: e.to_string(),763+                });764+            }765+            Ok(Some(status)) => break status,766+            Ok(None) => {767+                if start.elapsed() > GIT_TIMEOUT {768+                    let _ = child.kill();769+                    let _ = child.wait();770+                    let _ = std::fs::remove_file(dest);771+                    return Err(GitError::Timeout(cmd_string(repo, &args)));772+                }773+                thread::sleep(Duration::from_millis(10));774+            }775+        }776+    };777+    let written = rx_copy778+        .recv_timeout(Duration::from_secs(5))779+        .unwrap_or_else(|_| Err(std::io::Error::other("copy thread hung")));780+    let stderr = rx_err781+        .recv_timeout(Duration::from_secs(5))782+        .unwrap_or_default();783+    if !status.success() || written.is_err() {784+        let _ = std::fs::remove_file(dest);785+        return Err(GitError::Failed {786+            cmd: cmd_string(repo, &args),787+            status: status.to_string(),788+            stderr: String::from_utf8_lossy(&stderr).trim().to_string(),789+        });790+    }791+    Ok(())792+}793+672794 /// Unified diff text for one commit `sha`.673795 ///674796 /// Binary diffs keep git's `Binary files ... differ` note (or gain a
tests/large_blob.rs+71 −1
diff --git a/tests/large_blob.rs b/tests/large_blob.rsnew file mode 100644index 0000000..00414c7--- /dev/null+++ b/tests/large_blob.rs@@ -0,0 +1,71 @@1+//! f0rge #66: build must not read every blob fully into memory. A blob2+//! over the inline cap is streamed to `raw/`; its page is a placeholder3+//! and the build still succeeds.4+5+use std::path::{Path, PathBuf};6+use std::process::Command;7+8+fn git(repo: &Path, args: &[&str]) {9+    let st = Command::new("git")10+        .arg("-C")11+        .arg(repo)12+        .args(args)13+        .status()14+        .expect("spawn git");15+    assert!(st.success(), "git {args:?} failed");16+}17+18+fn fresh_dir(prefix: &str) -> PathBuf {19+    let dir = std::env::temp_dir().join(format!(20+        "hu13-large-{prefix}-{}-{}",21+        std::process::id(),22+        std::time::SystemTime::now()23+            .duration_since(std::time::UNIX_EPOCH)24+            .map(|d| d.as_nanos())25+            .unwrap_or(0)26+    ));27+    std::fs::create_dir_all(&dir).unwrap();28+    dir29+}30+31+#[test]32+fn oversized_blob_is_streamed_and_never_aborts() {33+    let repo = fresh_dir("repo");34+    let out = fresh_dir("out");35+    git(&repo, &["init", "-b", "main"]);36+    git(&repo, &["config", "user.name", "hu13-test"]);37+    git(&repo, &["config", "user.email", "[email protected]"]);38+    git(&repo, &["config", "commit.gpgsign", "false"]);39+    std::fs::write(repo.join("README.md"), "# large blob\n").unwrap();40+    // 12 MiB of text: above MAX_INLINE_BLOB (8 MiB), compressible enough41+    // for the test to stay fast.42+    let big = vec![b'a'; 12 * 1024 * 1024];43+    std::fs::write(repo.join("big.txt"), &big).unwrap();44+    git(&repo, &["add", "-A"]);45+    git(&repo, &["commit", "-m", "big"]);46+47+    let produced = Command::new(env!("CARGO_BIN_EXE_hu13"))48+        .arg("build")49+        .arg(&repo)50+        .arg("-o")51+        .arg(&out)52+        .output()53+        .expect("spawn hu13 build");54+    let stderr = String::from_utf8_lossy(&produced.stderr);55+    assert!(produced.status.success(), "build failed: {stderr}");56+57+    // Raw copy is byte-identical.58+    let raw = std::fs::read(out.join("raw/big.txt")).expect("raw big.txt");59+    assert_eq!(raw.len(), big.len());60+    assert!(raw.iter().all(|b| *b == b'a'));61+62+    // Page is a placeholder, not 12 MiB of spans.63+    let page = std::fs::read_to_string(out.join("files/big.txt.html")).expect("page");64+    assert!(page.contains("too large to preview"), "placeholder shown");65+    assert!(page.contains("download the raw file"));66+    assert!(page.len() < 200_000, "page stayed small: {}", page.len());67+68+    // A normal file still gets a real preview.69+    let readme = std::fs::read_to_string(out.join("files/README.md.html")).expect("readme");70+    assert!(readme.contains("<h1>README.md</h1>") || readme.contains("README.md"));71+}0-- 2.56.0