1 From aecf1e50b83fb1c2dc6a10a8a13cef2737b38104 Mon Sep 17 00:00:00 20012 From: hu13-test <[email protected]>3 Date: Tue, 27 Oct 2026 15:38:21 +00004 Subject: [PATCH] stream oversized blobs to raw/ instead of buffering (#66)5 6 A 400 MB blob made the build abort in the allocator ('memory7 allocation of 400000000 bytes failed'). Now blob_size() decides:8 blobs over MAX_INLINE_BLOB (8 MiB) go through show_to_file(),9 which copies git show stdout to raw/ in chunks, never enters the10 in-memory blob map, and renders a placeholder page. README lookup11 also skips candidates over 1 MiB. Partial files are removed when12 git fails or times out.13 ---14 src/build.rs | 104 +++++++++++++++++++++++++------------15 src/gitx.rs | 124 +++++++++++++++++++++++++++++++++++++++++++-16 tests/large_blob.rs | 71 +++++++++++++++++++++++++17 3 files changed, 264 insertions(+), 35 deletions(-)18 create mode 100644 tests/large_blob.rs19 20 diff --git a/src/build.rs b/src/build.rs21 index 90e6651..e3ba54a 10064422 --- a/src/build.rs23 +++ b/src/build.rs24 @@ -196,8 +196,13 @@ pub fn find_readme(repo: &Path, branch: &str) -> Option<(String, gitx::Blob)> {25 ".forge/wiki/README",26 ".forge/wiki/readme",27 ] {28 - if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) {29 - return Some((candidate.to_string(), blob));30 + let fits = gitx::blob_size(repo, branch, Path::new(candidate))31 + .map(|size| size <= README_MAX_BYTES)32 + .unwrap_or(false);33 + if fits {34 + if let Ok(blob) = gitx::show(repo, branch, Path::new(candidate)) {35 + return Some((candidate.to_string(), blob));36 + }37 }38 }39 None40 @@ -209,6 +214,17 @@ pub fn is_placeholder(bytes: &[u8]) -> bool {41 bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0)42 }43 44 +/// Blobs above this size are streamed straight to `raw/` instead of45 +/// being held in memory (f0rge #66). Reading every blob fully used to46 +/// abort the process in the allocator on a few-hundred-MB file; the47 +/// page for such a blob is a placeholder anyway, so only the raw copy48 +/// needs the bytes.49 +pub const MAX_INLINE_BLOB: u64 = 8 * 1024 * 1024;50 +51 +/// READMEs above this size are ignored (next candidate in the chain is52 +/// tried). Prevents a huge README from bloating every page's preview.53 +pub const README_MAX_BYTES: u64 = 1024 * 1024;54 +55 /// Bytes sniffed for the language-math binary check (first 8KiB).56 pub const BINARY_SNIFF_BYTES: usize = 8 * 1024;57 58 @@ -955,16 +971,27 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu59 wiki::page_count(repo) > 0,60 );61 62 - // Raw files first: byte-identical copies.63 + // Raw files first: byte-identical copies. Blobs over MAX_INLINE_BLOB64 + // are streamed to disk and never enter the in-memory map; their file65 + // pages become placeholders (f0rge #66).66 let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();67 + let mut oversize: BTreeMap<String, u64> = BTreeMap::new();68 for f in &files {69 - let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;70 let dest = out.join("raw").join(f);71 if let Some(parent) = dest.parent() {72 std::fs::create_dir_all(parent).map_err(|e| format!("hu13: create raw parent: {e}"))?;73 }74 - std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?;75 - blobs.insert(f.clone(), blob.bytes);76 + let size =77 + gitx::blob_size(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;78 + if size > MAX_INLINE_BLOB {79 + gitx::show_to_file(repo, &branch, Path::new(f), &dest)80 + .map_err(|e| format!("hu13: {e}"))?;81 + oversize.insert(f.clone(), size);82 + } else {83 + let blob = gitx::show(repo, &branch, Path::new(f)).map_err(|e| format!("hu13: {e}"))?;84 + std::fs::write(&dest, &blob.bytes).map_err(|e| format!("hu13: write raw {f}: {e}"))?;85 + blobs.insert(f.clone(), blob.bytes);86 + }87 }88 89 // Per-path pages (m6 code-gitea, f0rge #33): Gitea-style file90 @@ -1038,41 +1065,50 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu91 }92 93 for f in &files {94 - let bytes = blobs.get(f).cloned().unwrap_or_default();95 let mut body = breadcrumb(f);96 body.push_str(&format!("<h1>{}</h1>\n", esc(f)));97 - if is_placeholder(&bytes) {98 - body.push_str(&file_meta_bar(f, bytes.len() as u64, None));99 - let kind = if bytes.len() >= PREVIEW_MAX_BYTES {100 - format!("too large to preview ({} bytes)", bytes.len())101 - } else {102 - "binary file".to_string()103 - };104 + if let Some(&size) = oversize.get(f) {105 + body.push_str(&file_meta_bar(f, size, None));106 body.push_str(&format!(107 - "<p class=\"notice\">placeholder: {kind} — preview omitted. \108 + "<p class=\"notice\">placeholder: too large to preview ({size} bytes) — \109 <a href=\"/raw/{}\">download the raw file</a> instead.</p>\n",110 href_esc(f)111 ));112 - } else if is_markdown(f) {113 - let display = String::from_utf8_lossy(&bytes).into_owned();114 - body.push_str(&file_meta_bar(115 - f,116 - bytes.len() as u64,117 - Some(display.lines().count()),118 - ));119 - body.push_str(&markdown_to_html(&display));120 - body.push_str(&format!(121 - "<p class=\"muted\"><a href=\"/raw/{}\">view raw</a></p>\n",122 - href_esc(f)123 - ));124 } else {125 - let display = String::from_utf8_lossy(&bytes).into_owned();126 - body.push_str(&file_meta_bar(127 - f,128 - bytes.len() as u64,129 - Some(display.lines().count()),130 - ));131 - body.push_str(&render_text_pre(&display));132 + let bytes = blobs.get(f).cloned().unwrap_or_default();133 + if is_placeholder(&bytes) {134 + body.push_str(&file_meta_bar(f, bytes.len() as u64, None));135 + let kind = if bytes.len() >= PREVIEW_MAX_BYTES {136 + format!("too large to preview ({} bytes)", bytes.len())137 + } else {138 + "binary file".to_string()139 + };140 + body.push_str(&format!(141 + "<p class=\"notice\">placeholder: {kind} — preview omitted. \142 + <a href=\"/raw/{}\">download the raw file</a> instead.</p>\n",143 + href_esc(f)144 + ));145 + } else if is_markdown(f) {146 + let display = String::from_utf8_lossy(&bytes).into_owned();147 + body.push_str(&file_meta_bar(148 + f,149 + bytes.len() as u64,150 + Some(display.lines().count()),151 + ));152 + body.push_str(&markdown_to_html(&display));153 + body.push_str(&format!(154 + "<p class=\"muted\"><a href=\"/raw/{}\">view raw</a></p>\n",155 + href_esc(f)156 + ));157 + } else {158 + let display = String::from_utf8_lossy(&bytes).into_owned();159 + body.push_str(&file_meta_bar(160 + f,161 + bytes.len() as u64,162 + Some(display.lines().count()),163 + ));164 + body.push_str(&render_text_pre(&display));165 + }166 }167 let html = crate::layout::page_site_with_provenance(168 &format!("{f} — {name}"),169 diff --git a/src/gitx.rs b/src/gitx.rs170 index ac68f8f..8d09364 100644171 --- a/src/gitx.rs172 +++ b/src/gitx.rs173 @@ -7,7 +7,7 @@174 #![allow(dead_code)]175 176 use std::ffi::{OsStr, OsString};177 -use std::io::Read;178 +use std::io::{Read, Write};179 use std::path::{Path, PathBuf};180 use std::process::{Command, Stdio};181 use std::sync::mpsc;182 @@ -669,6 +669,128 @@ pub fn show(repo: &Path, branch: &str, path: &Path) -> Result<Blob, GitError> {183 })184 }185 186 +/// Size in bytes of `branch:path` without reading it (`git cat-file -s`).187 +///188 +/// Callers use this to avoid loading very large blobs into memory189 +/// (f0rge #66): a 400 MB blob used to abort the process in the190 +/// allocator.191 +pub fn blob_size(repo: &Path, branch: &str, path: &Path) -> Result<u64, GitError> {192 + let mut spec = OsString::from(branch);193 + spec.push(":");194 + spec.push(path.as_os_str());195 + let a_cat: &OsStr = OsStr::new("cat-file");196 + let a_size: &OsStr = OsStr::new("-s");197 + let args: Vec<&OsStr> = vec![a_cat, a_size, spec.as_os_str()];198 + let out = run_git(repo, &args)?;199 + if !out.status_ok {200 + let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();201 + return Err(GitError::NotFound(format!(202 + "`{branch}:{}` not found: {stderr}",203 + path.to_string_lossy()204 + )));205 + }206 + String::from_utf8_lossy(&out.stdout)207 + .trim()208 + .parse::<u64>()209 + .map_err(|e| GitError::Failed {210 + cmd: cmd_string(repo, &args),211 + status: "parse".to_string(),212 + stderr: format!("cat-file -s: {e}"),213 + })214 +}215 +216 +/// Stream `branch:path` to `dest` without buffering it in the process.217 +///218 +/// Used for blobs above the in-memory cap (f0rge #66). The destination219 +/// is removed again if git fails or the timeout fires, so a partial220 +/// file never survives a failed build.221 +pub fn show_to_file(repo: &Path, branch: &str, path: &Path, dest: &Path) -> Result<(), GitError> {222 + let mut spec = OsString::from(branch);223 + spec.push(":");224 + spec.push(path.as_os_str());225 + let a_show: &OsStr = OsStr::new("show");226 + let args: Vec<&OsStr> = vec![a_show, spec.as_os_str()];227 +228 + let mut cmd = Command::new("git");229 + cmd.arg("-C")230 + .arg(repo)231 + .args(&args)232 + .stdout(Stdio::piped())233 + .stderr(Stdio::piped());234 + let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?;235 + let stdout = child.stdout.take();236 + let stderr = child.stderr.take();237 +238 + let (tx_err, rx_err) = mpsc::channel::<Vec<u8>>();239 + if let Some(mut err) = stderr {240 + thread::spawn(move || {241 + let mut buf = Vec::new();242 + let _ = err.read_to_end(&mut buf);243 + let _ = tx_err.send(buf);244 + });245 + } else {246 + let _ = tx_err.send(Vec::new());247 + }248 +249 + let dest_owned = dest.to_path_buf();250 + let (tx_copy, rx_copy) = mpsc::channel::<std::io::Result<u64>>();251 + if let Some(mut out) = stdout {252 + thread::spawn(move || {253 + let file = match std::fs::File::create(&dest_owned) {254 + Ok(f) => f,255 + Err(e) => {256 + let _ = tx_copy.send(Err(e));257 + return;258 + }259 + };260 + let mut w = std::io::BufWriter::new(file);261 + let res = std::io::copy(&mut out, &mut w).and_then(|n| w.flush().map(|()| n));262 + let _ = tx_copy.send(res);263 + });264 + } else {265 + let _ = tx_copy.send(Err(std::io::Error::other("git show produced no stdout")));266 + }267 +268 + let start = Instant::now();269 + let status = loop {270 + match child.try_wait() {271 + Err(e) => {272 + let _ = std::fs::remove_file(dest);273 + return Err(GitError::Failed {274 + cmd: cmd_string(repo, &args),275 + status: "wait-error".to_string(),276 + stderr: e.to_string(),277 + });278 + }279 + Ok(Some(status)) => break status,280 + Ok(None) => {281 + if start.elapsed() > GIT_TIMEOUT {282 + let _ = child.kill();283 + let _ = child.wait();284 + let _ = std::fs::remove_file(dest);285 + return Err(GitError::Timeout(cmd_string(repo, &args)));286 + }287 + thread::sleep(Duration::from_millis(10));288 + }289 + }290 + };291 + let written = rx_copy292 + .recv_timeout(Duration::from_secs(5))293 + .unwrap_or_else(|_| Err(std::io::Error::other("copy thread hung")));294 + let stderr = rx_err295 + .recv_timeout(Duration::from_secs(5))296 + .unwrap_or_default();297 + if !status.success() || written.is_err() {298 + let _ = std::fs::remove_file(dest);299 + return Err(GitError::Failed {300 + cmd: cmd_string(repo, &args),301 + status: status.to_string(),302 + stderr: String::from_utf8_lossy(&stderr).trim().to_string(),303 + });304 + }305 + Ok(())306 +}307 +308 /// Unified diff text for one commit `sha`.309 ///310 /// Binary diffs keep git's `Binary files ... differ` note (or gain a311 diff --git a/tests/large_blob.rs b/tests/large_blob.rs312 new file mode 100644313 index 0000000..00414c7314 --- /dev/null315 +++ b/tests/large_blob.rs316 @@ -0,0 +1,71 @@317 +//! f0rge #66: build must not read every blob fully into memory. A blob318 +//! over the inline cap is streamed to `raw/`; its page is a placeholder319 +//! and the build still succeeds.320 +321 +use std::path::{Path, PathBuf};322 +use std::process::Command;323 +324 +fn git(repo: &Path, args: &[&str]) {325 + let st = Command::new("git")326 + .arg("-C")327 + .arg(repo)328 + .args(args)329 + .status()330 + .expect("spawn git");331 + assert!(st.success(), "git {args:?} failed");332 +}333 +334 +fn fresh_dir(prefix: &str) -> PathBuf {335 + let dir = std::env::temp_dir().join(format!(336 + "hu13-large-{prefix}-{}-{}",337 + std::process::id(),338 + std::time::SystemTime::now()339 + .duration_since(std::time::UNIX_EPOCH)340 + .map(|d| d.as_nanos())341 + .unwrap_or(0)342 + ));343 + std::fs::create_dir_all(&dir).unwrap();344 + dir345 +}346 +347 +#[test]348 +fn oversized_blob_is_streamed_and_never_aborts() {349 + let repo = fresh_dir("repo");350 + let out = fresh_dir("out");351 + git(&repo, &["init", "-b", "main"]);352 + git(&repo, &["config", "user.name", "hu13-test"]);353 + git(&repo, &["config", "user.email", "[email protected]"]);354 + git(&repo, &["config", "commit.gpgsign", "false"]);355 + std::fs::write(repo.join("README.md"), "# large blob\n").unwrap();356 + // 12 MiB of text: above MAX_INLINE_BLOB (8 MiB), compressible enough357 + // for the test to stay fast.358 + let big = vec![b'a'; 12 * 1024 * 1024];359 + std::fs::write(repo.join("big.txt"), &big).unwrap();360 + git(&repo, &["add", "-A"]);361 + git(&repo, &["commit", "-m", "big"]);362 +363 + let produced = Command::new(env!("CARGO_BIN_EXE_hu13"))364 + .arg("build")365 + .arg(&repo)366 + .arg("-o")367 + .arg(&out)368 + .output()369 + .expect("spawn hu13 build");370 + let stderr = String::from_utf8_lossy(&produced.stderr);371 + assert!(produced.status.success(), "build failed: {stderr}");372 +373 + // Raw copy is byte-identical.374 + let raw = std::fs::read(out.join("raw/big.txt")).expect("raw big.txt");375 + assert_eq!(raw.len(), big.len());376 + assert!(raw.iter().all(|b| *b == b'a'));377 +378 + // Page is a placeholder, not 12 MiB of spans.379 + let page = std::fs::read_to_string(out.join("files/big.txt.html")).expect("page");380 + assert!(page.contains("too large to preview"), "placeholder shown");381 + assert!(page.contains("download the raw file"));382 + assert!(page.len() < 200_000, "page stayed small: {}", page.len());383 +384 + // A normal file still gets a real preview.385 + let readme = std::fs::read_to_string(out.join("files/README.md.html")).expect("readme");386 + assert!(readme.contains("<h1>README.md</h1>") || readme.contains("README.md"));387 +}388 -- 389 2.56.0390
/
•
•