#32 m6/lang-ignore

src/build.rs+234 −9
diff --git a/src/build.rs b/src/build.rsindex f461204..d70dc3f 100644--- a/src/build.rs+++ b/src/build.rs@@ -207,6 +207,50 @@ pub fn is_placeholder(bytes: &[u8]) -> bool {207207     bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0)208208 }209209 210+/// Bytes sniffed for the language-math binary check (first 8KiB).211+pub const BINARY_SNIFF_BYTES: usize = 8 * 1024;212+213+/// `true` when a blob is binary for language math: the first 8KiB214+/// contain a NUL byte (m6, f0rge issue #37). Binaries stay215+/// downloadable and keep placeholder pages (see [`is_placeholder`]);216+/// they are only excluded from the language mix, never from the217+/// tree, raw output, or snapshots.218+#[must_use]219+pub fn is_binary(bytes: &[u8]) -> bool {220+    bytes.iter().take(BINARY_SNIFF_BYTES).any(|b| *b == 0)221+}222+223+/// `(path, size)` rows eligible for language math.224+///225+/// Skips `.forge/` (forge metadata: issues, PRs, patch diffs — not226+/// project code), gitignored paths (`ignored` is the set from227+/// [`gitx::ignored_paths`]) and binary blobs ([`is_binary`]).228+/// Everything kept is a real project text file; unmapped extensions229+/// still count (as `Other` via [`language_of`]). Percentages from230+/// [`compute_languages`] over this output therefore sum to counted231+/// text bytes only.232+#[must_use]233+pub fn lang_rows(234+    blobs: &std::collections::BTreeMap<String, Vec<u8>>,235+    ignored: &std::collections::HashSet<String>,236+) -> Vec<(String, u64)> {237+    blobs238+        .iter()239+        .filter(|(path, _)| !is_forge_meta(path))240+        .filter(|(path, _)| !ignored.contains(path.as_str()))241+        .filter(|(_, bytes)| !is_binary(bytes))242+        .map(|(path, bytes)| (path.clone(), bytes.len() as u64))243+        .collect()244+}245+246+/// `true` for `.forge/` metadata paths, which are excluded from247+/// language math: the forge (issues, PRs, diffs, runs) describes the248+/// project but is not its code (m6, f0rge issue #37).249+#[must_use]250+pub fn is_forge_meta(path: &str) -> bool {251+    path == ".forge" || path.starts_with(".forge/")252+}253+210254 /// `true` for markdown-rendered paths (`.md`/`.markdown`, any case).211255 #[must_use]212256 pub fn is_markdown(path: &str) -> bool {@@ -420,10 +464,13 @@ pub struct LangStat {420464 421465 /// Group `(path, bytes)` rows into language shares (bytes share).422466 ///423-/// Total is all rows passed in (unknown extensions count as `Other`).424-/// Languages under 1% merge into `Other`; output sorts bytes-desc.425-/// Empty input (or zero total) yields an empty vec (caller hides the426-/// bar and shows the no-languages note instead).467+/// Callers pass pre-filtered rows (see [`lang_rows`]): gitignored468+/// paths and binary blobs are already excluded, so the total here is469+/// counted text bytes only and `Other` holds just real text files470+/// with unmapped extensions. Languages under 1% merge into `Other`;471+/// output sorts bytes-desc. Empty input (or zero total) yields an472+/// empty vec (caller hides the bar and shows the no-languages note473+/// instead).427474 #[must_use]428475 pub fn compute_languages(files: &[(String, u64)]) -> Vec<LangStat> {429476     use std::collections::BTreeMap;@@ -465,7 +512,9 @@ pub fn compute_languages(files: &[(String, u64)]) -> Vec<LangStat> {465512 466513 /// Render the stacked language bar + legend (pure build-time, no JS).467514 ///468-/// `files` is `(path, bytes)` for the whole tree. Returns a515+/// `files` is pre-filtered `(path, bytes)` text rows (see [`lang_rows`]:516+/// gitignored and binary bytes never reach here, so `Other` is real517+/// text with unmapped extensions only). Returns a469518 /// `<section class="langs">` with a Tokyo Night stacked bar and a470519 /// `%` legend when at least one code file exists (a file whose471520 /// extension maps to a known language); otherwise a muted@@ -983,11 +1032,15 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu9831032     // Single column on mobile; tree+readme sit side by side only at >=768px.9841033     // Code tab file TABLE (m6 code-gitea, f0rge #33, Gitea-style) plus the9851034     // language mix bar (bytes share, Tokyo Night hues, no JS).1035+    // Language math (m6, f0rge #37): tracked files only (via gitx, so1036+    // untracked/ignored junk never counts), minus force-added paths1037+    // that still match gitignore, minus binary blobs (NUL in first1038+    // 8KiB). `Other` is real text with unmapped extensions only.1039+    // Ignore lookup never fails the build (degrades to no ignores).9861040     let root_entries = dir_entries.get("").cloned().unwrap_or_default();987-    let lang_files: Vec<(String, u64)> = blobs988-        .iter()989-        .map(|(k, v)| (k.clone(), v.len() as u64))990-        .collect();1041+    let ignored: std::collections::HashSet<String> =1042+        gitx::ignored_paths(repo, &files).unwrap_or_default();1043+    let lang_files: Vec<(String, u64)> = lang_rows(&blobs, &ignored);9911044     let lang_section = language_bar(&lang_files);9921045     body.push_str("<div class=\"cols\"><section><h2>Files</h2>\n");9931046     body.push_str(&file_table_html("", &root_entries, None, &history));@@ -1851,4 +1904,176 @@ mod tests {18511904         let tar_pos = index.find("repo.tar.gz").expect("tar pos");18521905         assert!(bundle_pos < tar_pos, "bundle before tarball");18531906     }1907+1908+    #[test]1909+    fn is_binary_sniffs_first_8kib() {1910+        // f0rge #37: NUL in the first 8KiB means binary for math.1911+        assert!(!is_binary(&[]), "empty is text");1912+        assert!(!is_binary(b"fn main() {}\n"), "plain text");1913+        assert!(is_binary(b"ab\x00cd"), "early NUL");1914+        assert!(!is_binary(&[b'a'; 100]), "no NUL anywhere");1915+        // NUL at the last sniffed byte still counts; just past it does not.1916+        let mut edge_in = vec![b'a'; BINARY_SNIFF_BYTES];1917+        edge_in[BINARY_SNIFF_BYTES - 1] = 0;1918+        assert!(is_binary(&edge_in), "NUL at byte 8191");1919+        let mut edge_out = vec![b'a'; BINARY_SNIFF_BYTES + 1];1920+        edge_out[BINARY_SNIFF_BYTES] = 0;1921+        assert!(!is_binary(&edge_out), "NUL past 8KiB ignored");1922+        assert_eq!(BINARY_SNIFF_BYTES, 8 * 1024, "8KiB sniff window");1923+    }1924+1925+    #[test]1926+    fn lang_rows_skips_ignored_and_binaries() {1927+        // Pure unit: ignored paths + NUL blobs drop out, real text1928+        // (even unmapped extensions) stays for `Other`.1929+        use std::collections::{BTreeMap, HashSet};1930+        let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();1931+        blobs.insert("a.rs".to_string(), b"fn a() {}\n".to_vec());1932+        blobs.insert("notes.txt".to_string(), b"hello\n".to_vec());1933+        blobs.insert("blob.bin".to_string(), b"ab\x00cd".to_vec());1934+        blobs.insert("ignored/s.rs".to_string(), b"fn s() {}\n".to_vec());1935+        let ignored: HashSet<String> = ["ignored/s.rs".to_string()].into_iter().collect();1936+        let mut rows = lang_rows(&blobs, &ignored);1937+        rows.sort();1938+        let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect();1939+        assert_eq!(paths, vec!["a.rs", "notes.txt"], "only text kept");1940+        // Sizes are exact byte lengths.1941+        let size = |name: &str| rows.iter().find(|(p, _)| p == name).unwrap().1;1942+        assert_eq!(size("a.rs"), b"fn a() {}\n".len() as u64);1943+        assert_eq!(size("notes.txt"), b"hello\n".len() as u64);1944+    }1945+1946+    #[test]1947+    fn lang_rows_skips_forge_metadata() {1948+        // f0rge #37 follow-up: .forge/ (issues, PR diffs, runs) is not1949+        // project code and stays out of the mix entirely.1950+        use std::collections::{BTreeMap, HashSet};1951+        let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();1952+        blobs.insert("src/main.rs".to_string(), b"fn main() {}\n".to_vec());1953+        blobs.insert(1954+            ".forge/prs/uuid/pr.md".to_string(),1955+            b"---\nid: x\n---\nbody\n".to_vec(),1956+        );1957+        blobs.insert(1958+            ".forge/prs/uuid/patch.diff".to_string(),1959+            b"diff --git a\n+line\n".to_vec(),1960+        );1961+        let rows = lang_rows(&blobs, &HashSet::new());1962+        let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect();1963+        assert_eq!(paths, vec!["src/main.rs"], "forge metadata excluded");1964+    }1965+1966+    #[test]1967+    fn language_math_excludes_gitignored_and_binaries() {1968+        // f0rge #37 rubric: fixture with a gitignored dir + a binary +1969+        // code files; ignored/binary bytes are excluded and the1970+        // legend percentages sum to counted text only.1971+        let dir = mktemp("lang-ignore");1972+        git(&dir, &["init", "-b", "main"]);1973+        git(&dir, &["config", "user.name", "hu13-test"]);1974+        git(&dir, &["config", "user.email", "[email protected]"]);1975+        git(&dir, &["config", "commit.gpgsign", "false"]);1976+        std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap();1977+        std::fs::create_dir_all(dir.join("src")).unwrap();1978+        let rs = "fn a() {}\n".repeat(20);1979+        let py = "print(1)\n".repeat(20);1980+        std::fs::write(dir.join("src").join("a.rs"), &rs).unwrap();1981+        std::fs::write(dir.join("src").join("b.py"), &py).unwrap();1982+        std::fs::write(dir.join("notes.txt"), "plain notes\n").unwrap();1983+        std::fs::create_dir_all(dir.join("assets")).unwrap();1984+        let mut bin: Vec<u8> = (0u8..=255).collect();1985+        bin.extend_from_slice(b"\x00\x01binary tail");1986+        std::fs::write(dir.join("assets").join("blob.bin"), &bin).unwrap();1987+        std::fs::create_dir_all(dir.join("ignored")).unwrap();1988+        // Large ignored code file: would dominate the bar if counted.1989+        std::fs::write(1990+            dir.join("ignored").join("secret.rs"),1991+            "fn s() {}\n".repeat(500),1992+        )1993+        .unwrap();1994+        std::fs::write(dir.join("debug.log"), "log line\n".repeat(500)).unwrap();1995+        git(&dir, &["add", "-A"]);1996+        git(&dir, &["commit", "-m", "base"]);1997+        // Force-add so the ignored paths are tracked (hu13 lists tracked1998+        // files); the language filter must still drop them via --no-index.1999+        git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]);2000+        git(&dir, &["commit", "-m", "forced ignored"]);2001+2002+        // gitx layer: ignored subset is exactly the forced paths.2003+        let tracked = vec![2004+            "src/a.rs".to_string(),2005+            "src/b.py".to_string(),2006+            "notes.txt".to_string(),2007+            "assets/blob.bin".to_string(),2008+            "ignored/secret.rs".to_string(),2009+            "debug.log".to_string(),2010+        ];2011+        let ignored = gitx::ignored_paths(&dir, &tracked).expect("ignored_paths");2012+        assert!(ignored.contains("ignored/secret.rs"), "dir rule");2013+        assert!(ignored.contains("debug.log"), "glob rule");2014+        assert!(!ignored.contains("src/a.rs"), "code kept");2015+        assert!(2016+            !ignored.contains("assets/blob.bin"),2017+            "binary kept here (binary rule drops it)"2018+        );2019+2020+        // Build layer: filtered rows are text only.2021+        let out = mktemp("out-lang-ignore");2022+        run_build(&dir, &out, &opts()).expect("build");2023+        let index = std::fs::read_to_string(out.join("index.html")).expect("index");2024+        let sec_start = index2025+            .find("<section class=\"langs\"")2026+            .expect("langs section");2027+        let sec_end = index[sec_start..].find("</section>").expect("langs end") + sec_start;2028+        let langs = &index[sec_start..sec_end];2029+        // Ignored + binary paths never appear in the math section2030+        // (they still have tree rows + raw downloads elsewhere).2031+        for needle in ["secret.rs", "debug.log", "blob.bin"] {2032+            assert!(2033+                !langs.contains(needle),2034+                "{needle} excluded from language math"2035+            );2036+        }2037+        assert!(langs.contains("Rust"), "rust counted");2038+        assert!(langs.contains("Python"), "python counted");2039+        // Percentages in the legend sum to ~100 (text-only total).2040+        // Parse only the legend list (the bar spans carry duplicate2041+        // `width:x%` values that must not be double-counted).2042+        let ul_start = langs2043+            .find("<ul class=\"lang-legend\">")2044+            .expect("legend list");2045+        let legend = &langs[ul_start..];2046+        let mut pct_sum = 0.0_f64;2047+        let mut count = 0_usize;2048+        let mut rest = legend;2049+        while let Some(i) = rest.find('%') {2050+            let start = rest[..i].rfind(['>', ' ']).map(|p| p + 1).unwrap_or(0);2051+            let num: f64 = rest[start..i].trim().parse().unwrap_or(f64::NAN);2052+            assert!(2053+                num.is_finite(),2054+                "legend % parses, got {:?}",2055+                &rest[start..i]2056+            );2057+            pct_sum += num;2058+            count += 1;2059+            rest = &rest[i + 1..];2060+        }2061+        assert!(count >= 2, "legend has shares, got {langs:?}");2062+        assert!(2063+            (pct_sum - 100.0).abs() < 0.2,2064+            "shares sum to 100, got {pct_sum} in {langs:?}"2065+        );2066+        // Binary is still downloadable with a placeholder page.2067+        let bin_page =2068+            std::fs::read_to_string(out.join("files/assets/blob.bin.html")).expect("bin page");2069+        assert!(bin_page.contains("placeholder"), "binary placeholder kept");2070+        assert!(2071+            out.join("raw/assets/blob.bin").exists(),2072+            "binary downloadable"2073+        );2074+        assert!(2075+            out.join("raw/ignored/secret.rs").exists(),2076+            "ignored still in raw"2077+        );2078+    }18542079 }
src/gitx.rs+190 −1
diff --git a/src/gitx.rs b/src/gitx.rsindex 76462fe..219d25a 100644--- a/src/gitx.rs+++ b/src/gitx.rs@@ -216,6 +216,96 @@ fn run_git(repo: &Path, args: &[&OsStr]) -> Result<GitOutput, GitError> {216216     }217217 }218218 219+fn run_git_stdin(220+    repo: &Path,221+    args: &[&OsStr],222+    stdin_bytes: Vec<u8>,223+) -> Result<GitOutput, GitError> {224+    use std::io::Write;225+    let mut cmd = Command::new("git");226+    cmd.arg("-C")227+        .arg(repo)228+        .args(args)229+        .stdin(Stdio::piped())230+        .stdout(Stdio::piped())231+        .stderr(Stdio::piped());232+    let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?;233+234+    if let Some(mut stdin) = child.stdin.take() {235+        let (tx_in, rx_in) = mpsc::channel::<Result<(), String>>();236+        thread::spawn(move || {237+            let r = stdin.write_all(&stdin_bytes).map_err(|e| e.to_string());238+            // Close stdin so git sees EOF even on partial writes.239+            drop(stdin);240+            let _ = tx_in.send(r);241+        });242+        // Wait briefly for the writer; a failure surfaces via git's exit.243+        let _ = rx_in.recv_timeout(Duration::from_secs(5));244+    }245+246+    let stdout_handle = child.stdout.take();247+    let stderr_handle = child.stderr.take();248+249+    let (tx_out, rx_out) = mpsc::channel::<Vec<u8>>();250+    let (tx_err, rx_err) = mpsc::channel::<Vec<u8>>();251+252+    if let Some(mut out) = stdout_handle {253+        let tx = tx_out;254+        thread::spawn(move || {255+            let mut buf = Vec::new();256+            let _ = out.read_to_end(&mut buf);257+            let _ = tx.send(buf);258+        });259+    } else {260+        let _ = tx_out.send(Vec::new());261+    }262+    if let Some(mut err) = stderr_handle {263+        let tx = tx_err;264+        thread::spawn(move || {265+            let mut buf = Vec::new();266+            let _ = err.read_to_end(&mut buf);267+            let _ = tx.send(buf);268+        });269+    } else {270+        let _ = tx_err.send(Vec::new());271+    }272+273+    let start = Instant::now();274+    loop {275+        match child.try_wait() {276+            Err(e) => {277+                return Err(GitError::Failed {278+                    cmd: cmd_string(repo, args),279+                    status: "wait-error".to_string(),280+                    stderr: e.to_string(),281+                })282+            }283+            Ok(Some(status)) => {284+                let stdout = rx_out285+                    .recv_timeout(Duration::from_secs(5))286+                    .unwrap_or_default();287+                let stderr = rx_err288+                    .recv_timeout(Duration::from_secs(5))289+                    .unwrap_or_default();290+                return Ok(GitOutput {291+                    status_ok: status.success(),292+                    status_str: status.to_string(),293+                    stdout,294+                    stderr,295+                });296+            }297+            Ok(None) => {298+                if start.elapsed() >= GIT_TIMEOUT {299+                    let _ = child.kill();300+                    let _ = child.wait();301+                    return Err(GitError::Timeout(cmd_string(repo, args)));302+                }303+                thread::sleep(Duration::from_millis(10));304+            }305+        }306+    }307+}308+219309 fn run_git_checked(repo: &Path, args: &[&OsStr]) -> Result<Vec<u8>, GitError> {220310     let out = run_git(repo, args)?;221311     if out.status_ok {@@ -485,6 +575,69 @@ pub fn ls_tree(repo: &Path, branch: &str, path: &str) -> Result<Vec<TreeEntry>,485575     Ok(entries)486576 }487577 578+/// Subset of repo-relative `paths` excluded by gitignore rules.579+///580+/// Runs `git check-ignore --no-index --stdin -z` (NUL-separated, so581+/// spaces/newlines in names survive). `--no-index` matters: by582+/// default git hides tracked files from exclude checks, but hu13583+/// lists tracked files via [`ls_tree`], so a force-added584+/// gitignored path must still be filtered from language math.585+/// Untracked/ignored junk never reaches [`ls_tree`] at all.586+///587+/// Returns the ignored subset (empty when nothing matches). Exit588+/// status 1 with empty output ("no matches") is success, not an589+/// error. Other git failures surface as [`GitError`]; callers590+/// building language math should degrade to "no ignores" rather591+/// than fail the build.592+///593+/// NUL bytes inside a path would corrupt the `-z` framing, so such594+/// paths are skipped on input (they never match output either).595+pub fn ignored_paths(596+    repo: &Path,597+    paths: &[String],598+) -> Result<std::collections::HashSet<String>, GitError> {599+    use std::collections::HashSet;600+    let clean: Vec<&str> = paths601+        .iter()602+        .map(String::as_str)603+        .filter(|p| !p.is_empty() && !p.contains('\0'))604+        .collect();605+    if clean.is_empty() {606+        return Ok(HashSet::new());607+    }608+    let mut stdin_bytes = Vec::new();609+    for p in &clean {610+        stdin_bytes.extend_from_slice(p.as_bytes());611+        stdin_bytes.push(0);612+    }613+    let a_check: &OsStr = OsStr::new("check-ignore");614+    let a_no_index: &OsStr = OsStr::new("--no-index");615+    let a_stdin: &OsStr = OsStr::new("--stdin");616+    let a_z: &OsStr = OsStr::new("-z");617+    let args: Vec<&OsStr> = vec![a_check, a_no_index, a_stdin, a_z];618+    let out = run_git_stdin(repo, &args, stdin_bytes)?;619+    if out.stdout.is_empty() {620+        // Exit 0 (all tracked, none ignored) or 1 (no matches): same result.621+        return Ok(HashSet::new());622+    }623+    if !out.status_ok {624+        let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();625+        return Err(GitError::Failed {626+            cmd: cmd_string(repo, &args),627+            status: out.status_str,628+            stderr,629+        });630+    }631+    let mut set = HashSet::new();632+    for chunk in out.stdout.split(|b| *b == 0) {633+        if chunk.is_empty() {634+            continue;635+        }636+        set.insert(String::from_utf8_lossy(chunk).into_owned());637+    }638+    Ok(set)639+}640+488641 /// Byte-exact file content for `branch:path`.489642 ///490643 /// `path` is a repo-relative path; non-UTF8 paths work because it is@@ -885,4 +1038,41 @@ mod tests {8851038         assert!(extract.join("zfile.txt").exists());8861039         assert!(extract.join("bin.dat").exists());8871040     }1041+1042+    #[test]1043+    fn ignored_paths_needs_no_index_for_tracked() {1044+        // hu13 lists tracked files, so force-added gitignored paths1045+        // must still be reported (plain check-ignore hides tracked).1046+        let dir = mktemp("ignored");1047+        git(&dir, &["init", "-b", "main"]);1048+        git(&dir, &["config", "user.name", "hu13-test"]);1049+        git(&dir, &["config", "user.email", "[email protected]"]);1050+        git(&dir, &["config", "commit.gpgsign", "false"]);1051+        std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap();1052+        std::fs::create_dir_all(dir.join("ignored")).unwrap();1053+        std::fs::write(dir.join("ignored").join("secret.rs"), "fn s() {}\n").unwrap();1054+        std::fs::write(dir.join("debug.log"), "log\n").unwrap();1055+        std::fs::write(dir.join("src.rs"), "fn a() {}\n").unwrap();1056+        git(&dir, &["add", "-A"]);1057+        git(&dir, &["commit", "-m", "base"]);1058+        // Force-add the ignored paths so they are tracked like hu13 sees.1059+        git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]);1060+        git(&dir, &["commit", "-m", "forced"]);1061+        let paths = vec![1062+            "src.rs".to_string(),1063+            "ignored/secret.rs".to_string(),1064+            "debug.log".to_string(),1065+            ".gitignore".to_string(),1066+        ];1067+        let ignored = ignored_paths(&dir, &paths).expect("ignored_paths");1068+        assert!(1069+            ignored.contains("ignored/secret.rs"),1070+            "force-added dir match, got {ignored:?}"1071+        );1072+        assert!(ignored.contains("debug.log"), "glob match, got {ignored:?}");1073+        assert!(!ignored.contains("src.rs"), "code kept, got {ignored:?}");1074+        assert!(!ignored.contains(".gitignore"), "gitignore kept");1075+        // Empty input never touches git.1076+        assert!(ignored_paths(&dir, &[]).expect("empty").is_empty());1077+    }8881078 }889-- 2.55.0