From aecf1e50b83fb1c2dc6a10a8a13cef2737b38104 Mon Sep 17 00:00:00 2001 From: hu13-test Date: Tue, 27 Oct 2026 15:38:21 +0000 Subject: [PATCH] stream oversized blobs to raw/ instead of buffering (#66) A 400 MB blob made the build abort in the allocator ('memory allocation of 400000000 bytes failed'). Now blob_size() decides: blobs over MAX_INLINE_BLOB (8 MiB) go through show_to_file(), which copies git show stdout to raw/ in chunks, never enters the in-memory blob map, and renders a placeholder page. README lookup also skips candidates over 1 MiB. Partial files are removed when git fails or times out. --- src/build.rs | 104 +++++++++++++++++++++++++------------ src/gitx.rs | 124 +++++++++++++++++++++++++++++++++++++++++++- tests/large_blob.rs | 71 +++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 35 deletions(-) create mode 100644 tests/large_blob.rs diff --git a/src/build.rs b/src/build.rs index 90e6651..e3ba54a 100644 --- a/src/build.rs +++ b/src/build.rs @@ -196,8 +196,13 @@ pub fn find_readme(repo: &Path, branch: &str) -> Option<(String, gitx::Blob)> { ".forge/wiki/README", ".forge/wiki/readme", ] { - if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) { - return Some((candidate.to_string(), blob)); + let fits = gitx::blob_size(repo, branch, Path::new(candidate)) + .map(|size| size <= README_MAX_BYTES) + .unwrap_or(false); + if fits { + if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) { + return Some((candidate.to_string(), blob)); + } } } None @@ -209,6 +214,17 @@ pub fn is_placeholder(bytes: &[u8]) -> bool { bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0) } +/// Blobs above this size are streamed straight to `raw/` instead of +/// being held in memory (f0rge #66). Reading every blob fully used to +/// abort the process in the allocator on a few-hundred-MB file; the +/// page for such a blob is a placeholder anyway, so only the raw copy +/// needs the bytes. +pub const MAX_INLINE_BLOB: u64 = 8 * 1024 * 1024; + +/// READMEs above this size are ignored (next candidate in the chain is +/// tried). Prevents a huge README from bloating every page's preview. +pub const README_MAX_BYTES: u64 = 1024 * 1024; + /// Bytes sniffed for the language-math binary check (first 8KiB). pub const BINARY_SNIFF_BYTES: usize = 8 * 1024; @@ -955,16 +971,27 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result 0, ); - // Raw files first: byte-identical copies. + // Raw files first: byte-identical copies. Blobs over MAX_INLINE_BLOB + // are streamed to disk and never enter the in-memory map; their file + // pages become placeholders (f0rge #66). let mut blobs: BTreeMap> = BTreeMap::new(); + let mut oversize: BTreeMap = BTreeMap::new(); for f in &files { - let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?; let dest = out.join("raw").join(f); if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent).map_err(|e| format!("hu13: create raw parent: {e}"))?; } - std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?; - blobs.insert(f.clone(), blob.bytes); + let size = + gitx::blob_size(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?; + if size > MAX_INLINE_BLOB { + gitx::show_to_file(repo, &branch, Path::new(f), &dest) + .map_err(|e| format!("hu13: {e}"))?; + oversize.insert(f.clone(), size); + } else { + let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?; + std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?; + blobs.insert(f.clone(), blob.bytes); + } } // Per-path pages (m6 code-gitea, f0rge #33): Gitea-style file @@ -1038,41 +1065,50 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result{}\n", esc(f))); - if is_placeholder(&bytes) { - body.push_str(&file_meta_bar(f, bytes.len() as u64, None)); - let kind = if bytes.len() >= PREVIEW_MAX_BYTES { - format!("too large to preview ({} bytes)", bytes.len()) - } else { - "binary file".to_string() - }; + if let Some(&size) = oversize.get(f) { + body.push_str(&file_meta_bar(f, size, None)); body.push_str(&format!( - "

placeholder: {kind} — preview omitted. \ + "

placeholder: too large to preview ({size} bytes) — \ download the raw file instead.

\n", href_esc(f) )); - } else if is_markdown(f) { - let display = String::from_utf8_lossy(&bytes).into_owned(); - body.push_str(&file_meta_bar( - f, - bytes.len() as u64, - Some(display.lines().count()), - )); - body.push_str(&markdown_to_html(&display)); - body.push_str(&format!( - "

view raw

\n", - href_esc(f) - )); } else { - let display = String::from_utf8_lossy(&bytes).into_owned(); - body.push_str(&file_meta_bar( - f, - bytes.len() as u64, - Some(display.lines().count()), - )); - body.push_str(&render_text_pre(&display)); + let bytes = blobs.get(f).cloned().unwrap_or_default(); + if is_placeholder(&bytes) { + body.push_str(&file_meta_bar(f, bytes.len() as u64, None)); + let kind = if bytes.len() >= PREVIEW_MAX_BYTES { + format!("too large to preview ({} bytes)", bytes.len()) + } else { + "binary file".to_string() + }; + body.push_str(&format!( + "

placeholder: {kind} — preview omitted. \ + download the raw file instead.

\n", + href_esc(f) + )); + } else if is_markdown(f) { + let display = String::from_utf8_lossy(&bytes).into_owned(); + body.push_str(&file_meta_bar( + f, + bytes.len() as u64, + Some(display.lines().count()), + )); + body.push_str(&markdown_to_html(&display)); + body.push_str(&format!( + "

view raw

\n", + href_esc(f) + )); + } else { + let display = String::from_utf8_lossy(&bytes).into_owned(); + body.push_str(&file_meta_bar( + f, + bytes.len() as u64, + Some(display.lines().count()), + )); + body.push_str(&render_text_pre(&display)); + } } let html = crate::layout::page_site_with_provenance( &format!("{f} — {name}"), diff --git a/src/gitx.rs b/src/gitx.rs index ac68f8f..8d09364 100644 --- a/src/gitx.rs +++ b/src/gitx.rs @@ -7,7 +7,7 @@ #![allow(dead_code)] use std::ffi::{OsStr, OsString}; -use std::io::Read; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::mpsc; @@ -669,6 +669,128 @@ pub fn show(repo: &Path, branch: &str, path: &Path) -> Result { }) } +/// Size in bytes of `branch:path` without reading it (`git cat-file -s`). +/// +/// Callers use this to avoid loading very large blobs into memory +/// (f0rge #66): a 400 MB blob used to abort the process in the +/// allocator. +pub fn blob_size(repo: &Path, branch: &str, path: &Path) -> Result { + let mut spec = OsString::from(branch); + spec.push(":"); + spec.push(path.as_os_str()); + let a_cat: &OsStr = OsStr::new("cat-file"); + let a_size: &OsStr = OsStr::new("-s"); + let args: Vec<&OsStr> = vec![a_cat, a_size, spec.as_os_str()]; + let out = run_git(repo, &args)?; + if !out.status_ok { + let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string(); + return Err(GitError::NotFound(format!( + "`{branch}:{}` not found: {stderr}", + path.to_string_lossy() + ))); + } + String::from_utf8_lossy(&out.stdout) + .trim() + .parse::() + .map_err(|e| GitError::Failed { + cmd: cmd_string(repo, &args), + status: "parse".to_string(), + stderr: format!("cat-file -s: {e}"), + }) +} + +/// Stream `branch:path` to `dest` without buffering it in the process. +/// +/// Used for blobs above the in-memory cap (f0rge #66). The destination +/// is removed again if git fails or the timeout fires, so a partial +/// file never survives a failed build. +pub fn show_to_file(repo: &Path, branch: &str, path: &Path, dest: &Path) -> Result<(), GitError> { + let mut spec = OsString::from(branch); + spec.push(":"); + spec.push(path.as_os_str()); + let a_show: &OsStr = OsStr::new("show"); + let args: Vec<&OsStr> = vec![a_show, spec.as_os_str()]; + + let mut cmd = Command::new("git"); + cmd.arg("-C") + .arg(repo) + .args(&args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + + let (tx_err, rx_err) = mpsc::channel::>(); + if let Some(mut err) = stderr { + thread::spawn(move || { + let mut buf = Vec::new(); + let _ = err.read_to_end(&mut buf); + let _ = tx_err.send(buf); + }); + } else { + let _ = tx_err.send(Vec::new()); + } + + let dest_owned = dest.to_path_buf(); + let (tx_copy, rx_copy) = mpsc::channel::>(); + if let Some(mut out) = stdout { + thread::spawn(move || { + let file = match std::fs::File::create(&dest_owned) { + Ok(f) => f, + Err(e) => { + let _ = tx_copy.send(Err(e)); + return; + } + }; + let mut w = std::io::BufWriter::new(file); + let res = std::io::copy(&mut out, &mut w).and_then(|n| w.flush().map(|()| n)); + let _ = tx_copy.send(res); + }); + } else { + let _ = tx_copy.send(Err(std::io::Error::other("git show produced no stdout"))); + } + + let start = Instant::now(); + let status = loop { + match child.try_wait() { + Err(e) => { + let _ = std::fs::remove_file(dest); + return Err(GitError::Failed { + cmd: cmd_string(repo, &args), + status: "wait-error".to_string(), + stderr: e.to_string(), + }); + } + Ok(Some(status)) => break status, + Ok(None) => { + if start.elapsed() > GIT_TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_file(dest); + return Err(GitError::Timeout(cmd_string(repo, &args))); + } + thread::sleep(Duration::from_millis(10)); + } + } + }; + let written = rx_copy + .recv_timeout(Duration::from_secs(5)) + .unwrap_or_else(|_| Err(std::io::Error::other("copy thread hung"))); + let stderr = rx_err + .recv_timeout(Duration::from_secs(5)) + .unwrap_or_default(); + if !status.success() || written.is_err() { + let _ = std::fs::remove_file(dest); + return Err(GitError::Failed { + cmd: cmd_string(repo, &args), + status: status.to_string(), + stderr: String::from_utf8_lossy(&stderr).trim().to_string(), + }); + } + Ok(()) +} + /// Unified diff text for one commit `sha`. /// /// Binary diffs keep git's `Binary files ... differ` note (or gain a diff --git a/tests/large_blob.rs b/tests/large_blob.rs new file mode 100644 index 0000000..00414c7 --- /dev/null +++ b/tests/large_blob.rs @@ -0,0 +1,71 @@ +//! f0rge #66: build must not read every blob fully into memory. A blob +//! over the inline cap is streamed to `raw/`; its page is a placeholder +//! and the build still succeeds. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn git(repo: &Path, args: &[&str]) { + let st = Command::new("git") + .arg("-C") + .arg(repo) + .args(args) + .status() + .expect("spawn git"); + assert!(st.success(), "git {args:?} failed"); +} + +fn fresh_dir(prefix: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "hu13-large-{prefix}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +#[test] +fn oversized_blob_is_streamed_and_never_aborts() { + let repo = fresh_dir("repo"); + let out = fresh_dir("out"); + git(&repo, &["init", "-b", "main"]); + git(&repo, &["config", "user.name", "hu13-test"]); + git(&repo, &["config", "user.email", "hu13@test.local"]); + git(&repo, &["config", "commit.gpgsign", "false"]); + std::fs::write(repo.join("README.md"), "# large blob\n").unwrap(); + // 12 MiB of text: above MAX_INLINE_BLOB (8 MiB), compressible enough + // for the test to stay fast. + let big = vec![b'a'; 12 * 1024 * 1024]; + std::fs::write(repo.join("big.txt"), &big).unwrap(); + git(&repo, &["add", "-A"]); + git(&repo, &["commit", "-m", "big"]); + + let produced = Command::new(env!("CARGO_BIN_EXE_hu13")) + .arg("build") + .arg(&repo) + .arg("-o") + .arg(&out) + .output() + .expect("spawn hu13 build"); + let stderr = String::from_utf8_lossy(&produced.stderr); + assert!(produced.status.success(), "build failed: {stderr}"); + + // Raw copy is byte-identical. + let raw = std::fs::read(out.join("raw/big.txt")).expect("raw big.txt"); + assert_eq!(raw.len(), big.len()); + assert!(raw.iter().all(|b| *b == b'a')); + + // Page is a placeholder, not 12 MiB of spans. + let page = std::fs::read_to_string(out.join("files/big.txt.html")).expect("page"); + assert!(page.contains("too large to preview"), "placeholder shown"); + assert!(page.contains("download the raw file")); + assert!(page.len() < 200_000, "page stayed small: {}", page.len()); + + // A normal file still gets a real preview. + let readme = std::fs::read_to_string(out.join("files/README.md.html")).expect("readme"); + assert!(readme.contains("

README.md

") || readme.contains("README.md")); +} -- 2.56.0