From d32b065c8176e94a7d087dd01b41dc5d7baf605b Mon Sep 17 00:00:00 2001 From: gl0w Date: Thu, 10 Sep 2026 08:22:37 +0100 Subject: [PATCH] m6/lang-ignore: gitignore+binaries+forge-meta out of mix --- src/build.rs | 243 +++++++++++++++++++++++++++++++++++++++++++++++++-- src/gitx.rs | 190 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 424 insertions(+), 9 deletions(-) diff --git a/src/build.rs b/src/build.rs index f461204..d70dc3f 100644 --- a/src/build.rs +++ b/src/build.rs @@ -207,6 +207,50 @@ pub fn is_placeholder(bytes: &[u8]) -> bool { bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0) } +/// Bytes sniffed for the language-math binary check (first 8KiB). +pub const BINARY_SNIFF_BYTES: usize = 8 * 1024; + +/// `true` when a blob is binary for language math: the first 8KiB +/// contain a NUL byte (m6, f0rge issue #37). Binaries stay +/// downloadable and keep placeholder pages (see [`is_placeholder`]); +/// they are only excluded from the language mix, never from the +/// tree, raw output, or snapshots. +#[must_use] +pub fn is_binary(bytes: &[u8]) -> bool { + bytes.iter().take(BINARY_SNIFF_BYTES).any(|b| *b == 0) +} + +/// `(path, size)` rows eligible for language math. +/// +/// Skips `.forge/` (forge metadata: issues, PRs, patch diffs — not +/// project code), gitignored paths (`ignored` is the set from +/// [`gitx::ignored_paths`]) and binary blobs ([`is_binary`]). +/// Everything kept is a real project text file; unmapped extensions +/// still count (as `Other` via [`language_of`]). Percentages from +/// [`compute_languages`] over this output therefore sum to counted +/// text bytes only. +#[must_use] +pub fn lang_rows( + blobs: &std::collections::BTreeMap>, + ignored: &std::collections::HashSet, +) -> Vec<(String, u64)> { + blobs + .iter() + .filter(|(path, _)| !is_forge_meta(path)) + .filter(|(path, _)| !ignored.contains(path.as_str())) + .filter(|(_, bytes)| !is_binary(bytes)) + .map(|(path, bytes)| (path.clone(), bytes.len() as u64)) + .collect() +} + +/// `true` for `.forge/` metadata paths, which are excluded from +/// language math: the forge (issues, PRs, diffs, runs) describes the +/// project but is not its code (m6, f0rge issue #37). +#[must_use] +pub fn is_forge_meta(path: &str) -> bool { + path == ".forge" || path.starts_with(".forge/") +} + /// `true` for markdown-rendered paths (`.md`/`.markdown`, any case). #[must_use] pub fn is_markdown(path: &str) -> bool { @@ -420,10 +464,13 @@ pub struct LangStat { /// Group `(path, bytes)` rows into language shares (bytes share). /// -/// Total is all rows passed in (unknown extensions count as `Other`). -/// Languages under 1% merge into `Other`; output sorts bytes-desc. -/// Empty input (or zero total) yields an empty vec (caller hides the -/// bar and shows the no-languages note instead). +/// Callers pass pre-filtered rows (see [`lang_rows`]): gitignored +/// paths and binary blobs are already excluded, so the total here is +/// counted text bytes only and `Other` holds just real text files +/// with unmapped extensions. Languages under 1% merge into `Other`; +/// output sorts bytes-desc. Empty input (or zero total) yields an +/// empty vec (caller hides the bar and shows the no-languages note +/// instead). #[must_use] pub fn compute_languages(files: &[(String, u64)]) -> Vec { use std::collections::BTreeMap; @@ -465,7 +512,9 @@ pub fn compute_languages(files: &[(String, u64)]) -> Vec { /// Render the stacked language bar + legend (pure build-time, no JS). /// -/// `files` is `(path, bytes)` for the whole tree. Returns a +/// `files` is pre-filtered `(path, bytes)` text rows (see [`lang_rows`]: +/// gitignored and binary bytes never reach here, so `Other` is real +/// text with unmapped extensions only). Returns a /// `
` with a Tokyo Night stacked bar and a /// `%` legend when at least one code file exists (a file whose /// extension maps to a known language); otherwise a muted @@ -983,11 +1032,15 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result=768px. // Code tab file TABLE (m6 code-gitea, f0rge #33, Gitea-style) plus the // language mix bar (bytes share, Tokyo Night hues, no JS). + // Language math (m6, f0rge #37): tracked files only (via gitx, so + // untracked/ignored junk never counts), minus force-added paths + // that still match gitignore, minus binary blobs (NUL in first + // 8KiB). `Other` is real text with unmapped extensions only. + // Ignore lookup never fails the build (degrades to no ignores). let root_entries = dir_entries.get("").cloned().unwrap_or_default(); - let lang_files: Vec<(String, u64)> = blobs - .iter() - .map(|(k, v)| (k.clone(), v.len() as u64)) - .collect(); + let ignored: std::collections::HashSet = + gitx::ignored_paths(repo, &files).unwrap_or_default(); + let lang_files: Vec<(String, u64)> = lang_rows(&blobs, &ignored); let lang_section = language_bar(&lang_files); body.push_str("

Files

\n"); body.push_str(&file_table_html("", &root_entries, None, &history)); @@ -1851,4 +1904,176 @@ mod tests { let tar_pos = index.find("repo.tar.gz").expect("tar pos"); assert!(bundle_pos < tar_pos, "bundle before tarball"); } + + #[test] + fn is_binary_sniffs_first_8kib() { + // f0rge #37: NUL in the first 8KiB means binary for math. + assert!(!is_binary(&[]), "empty is text"); + assert!(!is_binary(b"fn main() {}\n"), "plain text"); + assert!(is_binary(b"ab\x00cd"), "early NUL"); + assert!(!is_binary(&[b'a'; 100]), "no NUL anywhere"); + // NUL at the last sniffed byte still counts; just past it does not. + let mut edge_in = vec![b'a'; BINARY_SNIFF_BYTES]; + edge_in[BINARY_SNIFF_BYTES - 1] = 0; + assert!(is_binary(&edge_in), "NUL at byte 8191"); + let mut edge_out = vec![b'a'; BINARY_SNIFF_BYTES + 1]; + edge_out[BINARY_SNIFF_BYTES] = 0; + assert!(!is_binary(&edge_out), "NUL past 8KiB ignored"); + assert_eq!(BINARY_SNIFF_BYTES, 8 * 1024, "8KiB sniff window"); + } + + #[test] + fn lang_rows_skips_ignored_and_binaries() { + // Pure unit: ignored paths + NUL blobs drop out, real text + // (even unmapped extensions) stays for `Other`. + use std::collections::{BTreeMap, HashSet}; + let mut blobs: BTreeMap> = BTreeMap::new(); + blobs.insert("a.rs".to_string(), b"fn a() {}\n".to_vec()); + blobs.insert("notes.txt".to_string(), b"hello\n".to_vec()); + blobs.insert("blob.bin".to_string(), b"ab\x00cd".to_vec()); + blobs.insert("ignored/s.rs".to_string(), b"fn s() {}\n".to_vec()); + let ignored: HashSet = ["ignored/s.rs".to_string()].into_iter().collect(); + let mut rows = lang_rows(&blobs, &ignored); + rows.sort(); + let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!(paths, vec!["a.rs", "notes.txt"], "only text kept"); + // Sizes are exact byte lengths. + let size = |name: &str| rows.iter().find(|(p, _)| p == name).unwrap().1; + assert_eq!(size("a.rs"), b"fn a() {}\n".len() as u64); + assert_eq!(size("notes.txt"), b"hello\n".len() as u64); + } + + #[test] + fn lang_rows_skips_forge_metadata() { + // f0rge #37 follow-up: .forge/ (issues, PR diffs, runs) is not + // project code and stays out of the mix entirely. + use std::collections::{BTreeMap, HashSet}; + let mut blobs: BTreeMap> = BTreeMap::new(); + blobs.insert("src/main.rs".to_string(), b"fn main() {}\n".to_vec()); + blobs.insert( + ".forge/prs/uuid/pr.md".to_string(), + b"---\nid: x\n---\nbody\n".to_vec(), + ); + blobs.insert( + ".forge/prs/uuid/patch.diff".to_string(), + b"diff --git a\n+line\n".to_vec(), + ); + let rows = lang_rows(&blobs, &HashSet::new()); + let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!(paths, vec!["src/main.rs"], "forge metadata excluded"); + } + + #[test] + fn language_math_excludes_gitignored_and_binaries() { + // f0rge #37 rubric: fixture with a gitignored dir + a binary + + // code files; ignored/binary bytes are excluded and the + // legend percentages sum to counted text only. + let dir = mktemp("lang-ignore"); + git(&dir, &["init", "-b", "main"]); + git(&dir, &["config", "user.name", "hu13-test"]); + git(&dir, &["config", "user.email", "hu13@test.local"]); + git(&dir, &["config", "commit.gpgsign", "false"]); + std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap(); + std::fs::create_dir_all(dir.join("src")).unwrap(); + let rs = "fn a() {}\n".repeat(20); + let py = "print(1)\n".repeat(20); + std::fs::write(dir.join("src").join("a.rs"), &rs).unwrap(); + std::fs::write(dir.join("src").join("b.py"), &py).unwrap(); + std::fs::write(dir.join("notes.txt"), "plain notes\n").unwrap(); + std::fs::create_dir_all(dir.join("assets")).unwrap(); + let mut bin: Vec = (0u8..=255).collect(); + bin.extend_from_slice(b"\x00\x01binary tail"); + std::fs::write(dir.join("assets").join("blob.bin"), &bin).unwrap(); + std::fs::create_dir_all(dir.join("ignored")).unwrap(); + // Large ignored code file: would dominate the bar if counted. + std::fs::write( + dir.join("ignored").join("secret.rs"), + "fn s() {}\n".repeat(500), + ) + .unwrap(); + std::fs::write(dir.join("debug.log"), "log line\n".repeat(500)).unwrap(); + git(&dir, &["add", "-A"]); + git(&dir, &["commit", "-m", "base"]); + // Force-add so the ignored paths are tracked (hu13 lists tracked + // files); the language filter must still drop them via --no-index. + git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]); + git(&dir, &["commit", "-m", "forced ignored"]); + + // gitx layer: ignored subset is exactly the forced paths. + let tracked = vec![ + "src/a.rs".to_string(), + "src/b.py".to_string(), + "notes.txt".to_string(), + "assets/blob.bin".to_string(), + "ignored/secret.rs".to_string(), + "debug.log".to_string(), + ]; + let ignored = gitx::ignored_paths(&dir, &tracked).expect("ignored_paths"); + assert!(ignored.contains("ignored/secret.rs"), "dir rule"); + assert!(ignored.contains("debug.log"), "glob rule"); + assert!(!ignored.contains("src/a.rs"), "code kept"); + assert!( + !ignored.contains("assets/blob.bin"), + "binary kept here (binary rule drops it)" + ); + + // Build layer: filtered rows are text only. + let out = mktemp("out-lang-ignore"); + run_build(&dir, &out, &opts()).expect("build"); + let index = std::fs::read_to_string(out.join("index.html")).expect("index"); + let sec_start = index + .find("
").expect("langs end") + sec_start; + let langs = &index[sec_start..sec_end]; + // Ignored + binary paths never appear in the math section + // (they still have tree rows + raw downloads elsewhere). + for needle in ["secret.rs", "debug.log", "blob.bin"] { + assert!( + !langs.contains(needle), + "{needle} excluded from language math" + ); + } + assert!(langs.contains("Rust"), "rust counted"); + assert!(langs.contains("Python"), "python counted"); + // Percentages in the legend sum to ~100 (text-only total). + // Parse only the legend list (the bar spans carry duplicate + // `width:x%` values that must not be double-counted). + let ul_start = langs + .find("
    ") + .expect("legend list"); + let legend = &langs[ul_start..]; + let mut pct_sum = 0.0_f64; + let mut count = 0_usize; + let mut rest = legend; + while let Some(i) = rest.find('%') { + let start = rest[..i].rfind(['>', ' ']).map(|p| p + 1).unwrap_or(0); + let num: f64 = rest[start..i].trim().parse().unwrap_or(f64::NAN); + assert!( + num.is_finite(), + "legend % parses, got {:?}", + &rest[start..i] + ); + pct_sum += num; + count += 1; + rest = &rest[i + 1..]; + } + assert!(count >= 2, "legend has shares, got {langs:?}"); + assert!( + (pct_sum - 100.0).abs() < 0.2, + "shares sum to 100, got {pct_sum} in {langs:?}" + ); + // Binary is still downloadable with a placeholder page. + let bin_page = + std::fs::read_to_string(out.join("files/assets/blob.bin.html")).expect("bin page"); + assert!(bin_page.contains("placeholder"), "binary placeholder kept"); + assert!( + out.join("raw/assets/blob.bin").exists(), + "binary downloadable" + ); + assert!( + out.join("raw/ignored/secret.rs").exists(), + "ignored still in raw" + ); + } } diff --git a/src/gitx.rs b/src/gitx.rs index 76462fe..219d25a 100644 --- a/src/gitx.rs +++ b/src/gitx.rs @@ -216,6 +216,96 @@ fn run_git(repo: &Path, args: &[&OsStr]) -> Result { } } +fn run_git_stdin( + repo: &Path, + args: &[&OsStr], + stdin_bytes: Vec, +) -> Result { + use std::io::Write; + let mut cmd = Command::new("git"); + cmd.arg("-C") + .arg(repo) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?; + + if let Some(mut stdin) = child.stdin.take() { + let (tx_in, rx_in) = mpsc::channel::>(); + thread::spawn(move || { + let r = stdin.write_all(&stdin_bytes).map_err(|e| e.to_string()); + // Close stdin so git sees EOF even on partial writes. + drop(stdin); + let _ = tx_in.send(r); + }); + // Wait briefly for the writer; a failure surfaces via git's exit. + let _ = rx_in.recv_timeout(Duration::from_secs(5)); + } + + let stdout_handle = child.stdout.take(); + let stderr_handle = child.stderr.take(); + + let (tx_out, rx_out) = mpsc::channel::>(); + let (tx_err, rx_err) = mpsc::channel::>(); + + if let Some(mut out) = stdout_handle { + let tx = tx_out; + thread::spawn(move || { + let mut buf = Vec::new(); + let _ = out.read_to_end(&mut buf); + let _ = tx.send(buf); + }); + } else { + let _ = tx_out.send(Vec::new()); + } + if let Some(mut err) = stderr_handle { + let tx = tx_err; + thread::spawn(move || { + let mut buf = Vec::new(); + let _ = err.read_to_end(&mut buf); + let _ = tx.send(buf); + }); + } else { + let _ = tx_err.send(Vec::new()); + } + + let start = Instant::now(); + loop { + match child.try_wait() { + Err(e) => { + return Err(GitError::Failed { + cmd: cmd_string(repo, args), + status: "wait-error".to_string(), + stderr: e.to_string(), + }) + } + Ok(Some(status)) => { + let stdout = rx_out + .recv_timeout(Duration::from_secs(5)) + .unwrap_or_default(); + let stderr = rx_err + .recv_timeout(Duration::from_secs(5)) + .unwrap_or_default(); + return Ok(GitOutput { + status_ok: status.success(), + status_str: status.to_string(), + stdout, + stderr, + }); + } + Ok(None) => { + if start.elapsed() >= GIT_TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + return Err(GitError::Timeout(cmd_string(repo, args))); + } + thread::sleep(Duration::from_millis(10)); + } + } + } +} + fn run_git_checked(repo: &Path, args: &[&OsStr]) -> Result, GitError> { let out = run_git(repo, args)?; if out.status_ok { @@ -485,6 +575,69 @@ pub fn ls_tree(repo: &Path, branch: &str, path: &str) -> Result, Ok(entries) } +/// Subset of repo-relative `paths` excluded by gitignore rules. +/// +/// Runs `git check-ignore --no-index --stdin -z` (NUL-separated, so +/// spaces/newlines in names survive). `--no-index` matters: by +/// default git hides tracked files from exclude checks, but hu13 +/// lists tracked files via [`ls_tree`], so a force-added +/// gitignored path must still be filtered from language math. +/// Untracked/ignored junk never reaches [`ls_tree`] at all. +/// +/// Returns the ignored subset (empty when nothing matches). Exit +/// status 1 with empty output ("no matches") is success, not an +/// error. Other git failures surface as [`GitError`]; callers +/// building language math should degrade to "no ignores" rather +/// than fail the build. +/// +/// NUL bytes inside a path would corrupt the `-z` framing, so such +/// paths are skipped on input (they never match output either). +pub fn ignored_paths( + repo: &Path, + paths: &[String], +) -> Result, GitError> { + use std::collections::HashSet; + let clean: Vec<&str> = paths + .iter() + .map(String::as_str) + .filter(|p| !p.is_empty() && !p.contains('\0')) + .collect(); + if clean.is_empty() { + return Ok(HashSet::new()); + } + let mut stdin_bytes = Vec::new(); + for p in &clean { + stdin_bytes.extend_from_slice(p.as_bytes()); + stdin_bytes.push(0); + } + let a_check: &OsStr = OsStr::new("check-ignore"); + let a_no_index: &OsStr = OsStr::new("--no-index"); + let a_stdin: &OsStr = OsStr::new("--stdin"); + let a_z: &OsStr = OsStr::new("-z"); + let args: Vec<&OsStr> = vec![a_check, a_no_index, a_stdin, a_z]; + let out = run_git_stdin(repo, &args, stdin_bytes)?; + if out.stdout.is_empty() { + // Exit 0 (all tracked, none ignored) or 1 (no matches): same result. + return Ok(HashSet::new()); + } + if !out.status_ok { + let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string(); + return Err(GitError::Failed { + cmd: cmd_string(repo, &args), + status: out.status_str, + stderr, + }); + } + let mut set = HashSet::new(); + for chunk in out.stdout.split(|b| *b == 0) { + if chunk.is_empty() { + continue; + } + set.insert(String::from_utf8_lossy(chunk).into_owned()); + } + Ok(set) +} + /// Byte-exact file content for `branch:path`. /// /// `path` is a repo-relative path; non-UTF8 paths work because it is @@ -885,4 +1038,41 @@ mod tests { assert!(extract.join("zfile.txt").exists()); assert!(extract.join("bin.dat").exists()); } + + #[test] + fn ignored_paths_needs_no_index_for_tracked() { + // hu13 lists tracked files, so force-added gitignored paths + // must still be reported (plain check-ignore hides tracked). + let dir = mktemp("ignored"); + git(&dir, &["init", "-b", "main"]); + git(&dir, &["config", "user.name", "hu13-test"]); + git(&dir, &["config", "user.email", "hu13@test.local"]); + git(&dir, &["config", "commit.gpgsign", "false"]); + std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap(); + std::fs::create_dir_all(dir.join("ignored")).unwrap(); + std::fs::write(dir.join("ignored").join("secret.rs"), "fn s() {}\n").unwrap(); + std::fs::write(dir.join("debug.log"), "log\n").unwrap(); + std::fs::write(dir.join("src.rs"), "fn a() {}\n").unwrap(); + git(&dir, &["add", "-A"]); + git(&dir, &["commit", "-m", "base"]); + // Force-add the ignored paths so they are tracked like hu13 sees. + git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]); + git(&dir, &["commit", "-m", "forced"]); + let paths = vec![ + "src.rs".to_string(), + "ignored/secret.rs".to_string(), + "debug.log".to_string(), + ".gitignore".to_string(), + ]; + let ignored = ignored_paths(&dir, &paths).expect("ignored_paths"); + assert!( + ignored.contains("ignored/secret.rs"), + "force-added dir match, got {ignored:?}" + ); + assert!(ignored.contains("debug.log"), "glob match, got {ignored:?}"); + assert!(!ignored.contains("src.rs"), "code kept, got {ignored:?}"); + assert!(!ignored.contains(".gitignore"), "gitignore kept"); + // Empty input never touches git. + assert!(ignored_paths(&dir, &[]).expect("empty").is_empty()); + } } -- 2.55.0