1 From d32b065c8176e94a7d087dd01b41dc5d7baf605b Mon Sep 17 00:00:00 20012 From: gl0w <gl0w@world>3 Date: Thu, 10 Sep 2026 08:22:37 +01004 Subject: [PATCH] m6/lang-ignore: gitignore+binaries+forge-meta out of mix5 6 ---7  src/build.rs | 243 +++++++++++++++++++++++++++++++++++++++++++++++++--8  src/gitx.rs  | 190 ++++++++++++++++++++++++++++++++++++++++9  2 files changed, 424 insertions(+), 9 deletions(-)10 11 diff --git a/src/build.rs b/src/build.rs12 index f461204..d70dc3f 10064413 --- a/src/build.rs14 +++ b/src/build.rs15 @@ -207,6 +207,50 @@ pub fn is_placeholder(bytes: &[u8]) -> bool {16      bytes.len() >= PREVIEW_MAX_BYTES || bytes.contains(&0)17  }18  19 +/// Bytes sniffed for the language-math binary check (first 8KiB).20 +pub const BINARY_SNIFF_BYTES: usize = 8 * 1024;21 +22 +/// `true` when a blob is binary for language math: the first 8KiB23 +/// contain a NUL byte (m6, f0rge issue #37). Binaries stay24 +/// downloadable and keep placeholder pages (see [`is_placeholder`]);25 +/// they are only excluded from the language mix, never from the26 +/// tree, raw output, or snapshots.27 +#[must_use]28 +pub fn is_binary(bytes: &[u8]) -> bool {29 +    bytes.iter().take(BINARY_SNIFF_BYTES).any(|b| *b == 0)30 +}31 +32 +/// `(path, size)` rows eligible for language math.33 +///34 +/// Skips `.forge/` (forge metadata: issues, PRs, patch diffs — not35 +/// project code), gitignored paths (`ignored` is the set from36 +/// [`gitx::ignored_paths`]) and binary blobs ([`is_binary`]).37 +/// Everything kept is a real project text file; unmapped extensions38 +/// still count (as `Other` via [`language_of`]). Percentages from39 +/// [`compute_languages`] over this output therefore sum to counted40 +/// text bytes only.41 +#[must_use]42 +pub fn lang_rows(43 +    blobs: &std::collections::BTreeMap<String, Vec<u8>>,44 +    ignored: &std::collections::HashSet<String>,45 +) -> Vec<(String, u64)> {46 +    blobs47 +        .iter()48 +        .filter(|(path, _)| !is_forge_meta(path))49 +        .filter(|(path, _)| !ignored.contains(path.as_str()))50 +        .filter(|(_, bytes)| !is_binary(bytes))51 +        .map(|(path, bytes)| (path.clone(), bytes.len() as u64))52 +        .collect()53 +}54 +55 +/// `true` for `.forge/` metadata paths, which are excluded from56 +/// language math: the forge (issues, PRs, diffs, runs) describes the57 +/// project but is not its code (m6, f0rge issue #37).58 +#[must_use]59 +pub fn is_forge_meta(path: &str) -> bool {60 +    path == ".forge" || path.starts_with(".forge/")61 +}62 +63  /// `true` for markdown-rendered paths (`.md`/`.markdown`, any case).64  #[must_use]65  pub fn is_markdown(path: &str) -> bool {66 @@ -420,10 +464,13 @@ pub struct LangStat {67  68  /// Group `(path, bytes)` rows into language shares (bytes share).69  ///70 -/// Total is all rows passed in (unknown extensions count as `Other`).71 -/// Languages under 1% merge into `Other`; output sorts bytes-desc.72 -/// Empty input (or zero total) yields an empty vec (caller hides the73 -/// bar and shows the no-languages note instead).74 +/// Callers pass pre-filtered rows (see [`lang_rows`]): gitignored75 +/// paths and binary blobs are already excluded, so the total here is76 +/// counted text bytes only and `Other` holds just real text files77 +/// with unmapped extensions. Languages under 1% merge into `Other`;78 +/// output sorts bytes-desc. Empty input (or zero total) yields an79 +/// empty vec (caller hides the bar and shows the no-languages note80 +/// instead).81  #[must_use]82  pub fn compute_languages(files: &[(String, u64)]) -> Vec<LangStat> {83      use std::collections::BTreeMap;84 @@ -465,7 +512,9 @@ pub fn compute_languages(files: &[(String, u64)]) -> Vec<LangStat> {85  86  /// Render the stacked language bar + legend (pure build-time, no JS).87  ///88 -/// `files` is `(path, bytes)` for the whole tree. Returns a89 +/// `files` is pre-filtered `(path, bytes)` text rows (see [`lang_rows`]:90 +/// gitignored and binary bytes never reach here, so `Other` is real91 +/// text with unmapped extensions only). Returns a92  /// `<section class="langs">` with a Tokyo Night stacked bar and a93  /// `%` legend when at least one code file exists (a file whose94  /// extension maps to a known language); otherwise a muted95 @@ -983,11 +1032,15 @@ pub fn run_build(repo: &Path, out: &Path, opts: &BuildOptions) -> Result<BuildSu96      // Single column on mobile; tree+readme sit side by side only at >=768px.97      // Code tab file TABLE (m6 code-gitea, f0rge #33, Gitea-style) plus the98      // language mix bar (bytes share, Tokyo Night hues, no JS).99 +    // Language math (m6, f0rge #37): tracked files only (via gitx, so100 +    // untracked/ignored junk never counts), minus force-added paths101 +    // that still match gitignore, minus binary blobs (NUL in first102 +    // 8KiB). `Other` is real text with unmapped extensions only.103 +    // Ignore lookup never fails the build (degrades to no ignores).104      let root_entries = dir_entries.get("").cloned().unwrap_or_default();105 -    let lang_files: Vec<(String, u64)> = blobs106 -        .iter()107 -        .map(|(k, v)| (k.clone(), v.len() as u64))108 -        .collect();109 +    let ignored: std::collections::HashSet<String> =110 +        gitx::ignored_paths(repo, &files).unwrap_or_default();111 +    let lang_files: Vec<(String, u64)> = lang_rows(&blobs, &ignored);112      let lang_section = language_bar(&lang_files);113      body.push_str("<div class=\"cols\"><section><h2>Files</h2>\n");114      body.push_str(&file_table_html("", &root_entries, None, &history));115 @@ -1851,4 +1904,176 @@ mod tests {116          let tar_pos = index.find("repo.tar.gz").expect("tar pos");117          assert!(bundle_pos < tar_pos, "bundle before tarball");118      }119 +120 +    #[test]121 +    fn is_binary_sniffs_first_8kib() {122 +        // f0rge #37: NUL in the first 8KiB means binary for math.123 +        assert!(!is_binary(&[]), "empty is text");124 +        assert!(!is_binary(b"fn main() {}\n"), "plain text");125 +        assert!(is_binary(b"ab\x00cd"), "early NUL");126 +        assert!(!is_binary(&[b'a'; 100]), "no NUL anywhere");127 +        // NUL at the last sniffed byte still counts; just past it does not.128 +        let mut edge_in = vec![b'a'; BINARY_SNIFF_BYTES];129 +        edge_in[BINARY_SNIFF_BYTES - 1] = 0;130 +        assert!(is_binary(&edge_in), "NUL at byte 8191");131 +        let mut edge_out = vec![b'a'; BINARY_SNIFF_BYTES + 1];132 +        edge_out[BINARY_SNIFF_BYTES] = 0;133 +        assert!(!is_binary(&edge_out), "NUL past 8KiB ignored");134 +        assert_eq!(BINARY_SNIFF_BYTES, 8 * 1024, "8KiB sniff window");135 +    }136 +137 +    #[test]138 +    fn lang_rows_skips_ignored_and_binaries() {139 +        // Pure unit: ignored paths + NUL blobs drop out, real text140 +        // (even unmapped extensions) stays for `Other`.141 +        use std::collections::{BTreeMap, HashSet};142 +        let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();143 +        blobs.insert("a.rs".to_string(), b"fn a() {}\n".to_vec());144 +        blobs.insert("notes.txt".to_string(), b"hello\n".to_vec());145 +        blobs.insert("blob.bin".to_string(), b"ab\x00cd".to_vec());146 +        blobs.insert("ignored/s.rs".to_string(), b"fn s() {}\n".to_vec());147 +        let ignored: HashSet<String> = ["ignored/s.rs".to_string()].into_iter().collect();148 +        let mut rows = lang_rows(&blobs, &ignored);149 +        rows.sort();150 +        let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect();151 +        assert_eq!(paths, vec!["a.rs", "notes.txt"], "only text kept");152 +        // Sizes are exact byte lengths.153 +        let size = |name: &str| rows.iter().find(|(p, _)| p == name).unwrap().1;154 +        assert_eq!(size("a.rs"), b"fn a() {}\n".len() as u64);155 +        assert_eq!(size("notes.txt"), b"hello\n".len() as u64);156 +    }157 +158 +    #[test]159 +    fn lang_rows_skips_forge_metadata() {160 +        // f0rge #37 follow-up: .forge/ (issues, PR diffs, runs) is not161 +        // project code and stays out of the mix entirely.162 +        use std::collections::{BTreeMap, HashSet};163 +        let mut blobs: BTreeMap<String, Vec<u8>> = BTreeMap::new();164 +        blobs.insert("src/main.rs".to_string(), b"fn main() {}\n".to_vec());165 +        blobs.insert(166 +            ".forge/prs/uuid/pr.md".to_string(),167 +            b"---\nid: x\n---\nbody\n".to_vec(),168 +        );169 +        blobs.insert(170 +            ".forge/prs/uuid/patch.diff".to_string(),171 +            b"diff --git a\n+line\n".to_vec(),172 +        );173 +        let rows = lang_rows(&blobs, &HashSet::new());174 +        let paths: Vec<&str> = rows.iter().map(|(p, _)| p.as_str()).collect();175 +        assert_eq!(paths, vec!["src/main.rs"], "forge metadata excluded");176 +    }177 +178 +    #[test]179 +    fn language_math_excludes_gitignored_and_binaries() {180 +        // f0rge #37 rubric: fixture with a gitignored dir + a binary +181 +        // code files; ignored/binary bytes are excluded and the182 +        // legend percentages sum to counted text only.183 +        let dir = mktemp("lang-ignore");184 +        git(&dir, &["init", "-b", "main"]);185 +        git(&dir, &["config", "user.name", "hu13-test"]);186 +        git(&dir, &["config", "user.email", "[email protected]"]);187 +        git(&dir, &["config", "commit.gpgsign", "false"]);188 +        std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap();189 +        std::fs::create_dir_all(dir.join("src")).unwrap();190 +        let rs = "fn a() {}\n".repeat(20);191 +        let py = "print(1)\n".repeat(20);192 +        std::fs::write(dir.join("src").join("a.rs"), &rs).unwrap();193 +        std::fs::write(dir.join("src").join("b.py"), &py).unwrap();194 +        std::fs::write(dir.join("notes.txt"), "plain notes\n").unwrap();195 +        std::fs::create_dir_all(dir.join("assets")).unwrap();196 +        let mut bin: Vec<u8> = (0u8..=255).collect();197 +        bin.extend_from_slice(b"\x00\x01binary tail");198 +        std::fs::write(dir.join("assets").join("blob.bin"), &bin).unwrap();199 +        std::fs::create_dir_all(dir.join("ignored")).unwrap();200 +        // Large ignored code file: would dominate the bar if counted.201 +        std::fs::write(202 +            dir.join("ignored").join("secret.rs"),203 +            "fn s() {}\n".repeat(500),204 +        )205 +        .unwrap();206 +        std::fs::write(dir.join("debug.log"), "log line\n".repeat(500)).unwrap();207 +        git(&dir, &["add", "-A"]);208 +        git(&dir, &["commit", "-m", "base"]);209 +        // Force-add so the ignored paths are tracked (hu13 lists tracked210 +        // files); the language filter must still drop them via --no-index.211 +        git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]);212 +        git(&dir, &["commit", "-m", "forced ignored"]);213 +214 +        // gitx layer: ignored subset is exactly the forced paths.215 +        let tracked = vec![216 +            "src/a.rs".to_string(),217 +            "src/b.py".to_string(),218 +            "notes.txt".to_string(),219 +            "assets/blob.bin".to_string(),220 +            "ignored/secret.rs".to_string(),221 +            "debug.log".to_string(),222 +        ];223 +        let ignored = gitx::ignored_paths(&dir, &tracked).expect("ignored_paths");224 +        assert!(ignored.contains("ignored/secret.rs"), "dir rule");225 +        assert!(ignored.contains("debug.log"), "glob rule");226 +        assert!(!ignored.contains("src/a.rs"), "code kept");227 +        assert!(228 +            !ignored.contains("assets/blob.bin"),229 +            "binary kept here (binary rule drops it)"230 +        );231 +232 +        // Build layer: filtered rows are text only.233 +        let out = mktemp("out-lang-ignore");234 +        run_build(&dir, &out, &opts()).expect("build");235 +        let index = std::fs::read_to_string(out.join("index.html")).expect("index");236 +        let sec_start = index237 +            .find("<section class=\"langs\"")238 +            .expect("langs section");239 +        let sec_end = index[sec_start..].find("</section>").expect("langs end") + sec_start;240 +        let langs = &index[sec_start..sec_end];241 +        // Ignored + binary paths never appear in the math section242 +        // (they still have tree rows + raw downloads elsewhere).243 +        for needle in ["secret.rs", "debug.log", "blob.bin"] {244 +            assert!(245 +                !langs.contains(needle),246 +                "{needle} excluded from language math"247 +            );248 +        }249 +        assert!(langs.contains("Rust"), "rust counted");250 +        assert!(langs.contains("Python"), "python counted");251 +        // Percentages in the legend sum to ~100 (text-only total).252 +        // Parse only the legend list (the bar spans carry duplicate253 +        // `width:x%` values that must not be double-counted).254 +        let ul_start = langs255 +            .find("<ul class=\"lang-legend\">")256 +            .expect("legend list");257 +        let legend = &langs[ul_start..];258 +        let mut pct_sum = 0.0_f64;259 +        let mut count = 0_usize;260 +        let mut rest = legend;261 +        while let Some(i) = rest.find('%') {262 +            let start = rest[..i].rfind(['>', ' ']).map(|p| p + 1).unwrap_or(0);263 +            let num: f64 = rest[start..i].trim().parse().unwrap_or(f64::NAN);264 +            assert!(265 +                num.is_finite(),266 +                "legend % parses, got {:?}",267 +                &rest[start..i]268 +            );269 +            pct_sum += num;270 +            count += 1;271 +            rest = &rest[i + 1..];272 +        }273 +        assert!(count >= 2, "legend has shares, got {langs:?}");274 +        assert!(275 +            (pct_sum - 100.0).abs() < 0.2,276 +            "shares sum to 100, got {pct_sum} in {langs:?}"277 +        );278 +        // Binary is still downloadable with a placeholder page.279 +        let bin_page =280 +            std::fs::read_to_string(out.join("files/assets/blob.bin.html")).expect("bin page");281 +        assert!(bin_page.contains("placeholder"), "binary placeholder kept");282 +        assert!(283 +            out.join("raw/assets/blob.bin").exists(),284 +            "binary downloadable"285 +        );286 +        assert!(287 +            out.join("raw/ignored/secret.rs").exists(),288 +            "ignored still in raw"289 +        );290 +    }291  }292 diff --git a/src/gitx.rs b/src/gitx.rs293 index 76462fe..219d25a 100644294 --- a/src/gitx.rs295 +++ b/src/gitx.rs296 @@ -216,6 +216,96 @@ fn run_git(repo: &Path, args: &[&OsStr]) -> Result<GitOutput, GitError> {297      }298  }299  300 +fn run_git_stdin(301 +    repo: &Path,302 +    args: &[&OsStr],303 +    stdin_bytes: Vec<u8>,304 +) -> Result<GitOutput, GitError> {305 +    use std::io::Write;306 +    let mut cmd = Command::new("git");307 +    cmd.arg("-C")308 +        .arg(repo)309 +        .args(args)310 +        .stdin(Stdio::piped())311 +        .stdout(Stdio::piped())312 +        .stderr(Stdio::piped());313 +    let mut child = cmd.spawn().map_err(|e| GitError::Spawn(e.to_string()))?;314 +315 +    if let Some(mut stdin) = child.stdin.take() {316 +        let (tx_in, rx_in) = mpsc::channel::<Result<(), String>>();317 +        thread::spawn(move || {318 +            let r = stdin.write_all(&stdin_bytes).map_err(|e| e.to_string());319 +            // Close stdin so git sees EOF even on partial writes.320 +            drop(stdin);321 +            let _ = tx_in.send(r);322 +        });323 +        // Wait briefly for the writer; a failure surfaces via git's exit.324 +        let _ = rx_in.recv_timeout(Duration::from_secs(5));325 +    }326 +327 +    let stdout_handle = child.stdout.take();328 +    let stderr_handle = child.stderr.take();329 +330 +    let (tx_out, rx_out) = mpsc::channel::<Vec<u8>>();331 +    let (tx_err, rx_err) = mpsc::channel::<Vec<u8>>();332 +333 +    if let Some(mut out) = stdout_handle {334 +        let tx = tx_out;335 +        thread::spawn(move || {336 +            let mut buf = Vec::new();337 +            let _ = out.read_to_end(&mut buf);338 +            let _ = tx.send(buf);339 +        });340 +    } else {341 +        let _ = tx_out.send(Vec::new());342 +    }343 +    if let Some(mut err) = stderr_handle {344 +        let tx = tx_err;345 +        thread::spawn(move || {346 +            let mut buf = Vec::new();347 +            let _ = err.read_to_end(&mut buf);348 +            let _ = tx.send(buf);349 +        });350 +    } else {351 +        let _ = tx_err.send(Vec::new());352 +    }353 +354 +    let start = Instant::now();355 +    loop {356 +        match child.try_wait() {357 +            Err(e) => {358 +                return Err(GitError::Failed {359 +                    cmd: cmd_string(repo, args),360 +                    status: "wait-error".to_string(),361 +                    stderr: e.to_string(),362 +                })363 +            }364 +            Ok(Some(status)) => {365 +                let stdout = rx_out366 +                    .recv_timeout(Duration::from_secs(5))367 +                    .unwrap_or_default();368 +                let stderr = rx_err369 +                    .recv_timeout(Duration::from_secs(5))370 +                    .unwrap_or_default();371 +                return Ok(GitOutput {372 +                    status_ok: status.success(),373 +                    status_str: status.to_string(),374 +                    stdout,375 +                    stderr,376 +                });377 +            }378 +            Ok(None) => {379 +                if start.elapsed() >= GIT_TIMEOUT {380 +                    let _ = child.kill();381 +                    let _ = child.wait();382 +                    return Err(GitError::Timeout(cmd_string(repo, args)));383 +                }384 +                thread::sleep(Duration::from_millis(10));385 +            }386 +        }387 +    }388 +}389 +390  fn run_git_checked(repo: &Path, args: &[&OsStr]) -> Result<Vec<u8>, GitError> {391      let out = run_git(repo, args)?;392      if out.status_ok {393 @@ -485,6 +575,69 @@ pub fn ls_tree(repo: &Path, branch: &str, path: &str) -> Result<Vec<TreeEntry>,394      Ok(entries)395  }396  397 +/// Subset of repo-relative `paths` excluded by gitignore rules.398 +///399 +/// Runs `git check-ignore --no-index --stdin -z` (NUL-separated, so400 +/// spaces/newlines in names survive). `--no-index` matters: by401 +/// default git hides tracked files from exclude checks, but hu13402 +/// lists tracked files via [`ls_tree`], so a force-added403 +/// gitignored path must still be filtered from language math.404 +/// Untracked/ignored junk never reaches [`ls_tree`] at all.405 +///406 +/// Returns the ignored subset (empty when nothing matches). Exit407 +/// status 1 with empty output ("no matches") is success, not an408 +/// error. Other git failures surface as [`GitError`]; callers409 +/// building language math should degrade to "no ignores" rather410 +/// than fail the build.411 +///412 +/// NUL bytes inside a path would corrupt the `-z` framing, so such413 +/// paths are skipped on input (they never match output either).414 +pub fn ignored_paths(415 +    repo: &Path,416 +    paths: &[String],417 +) -> Result<std::collections::HashSet<String>, GitError> {418 +    use std::collections::HashSet;419 +    let clean: Vec<&str> = paths420 +        .iter()421 +        .map(String::as_str)422 +        .filter(|p| !p.is_empty() && !p.contains('\0'))423 +        .collect();424 +    if clean.is_empty() {425 +        return Ok(HashSet::new());426 +    }427 +    let mut stdin_bytes = Vec::new();428 +    for p in &clean {429 +        stdin_bytes.extend_from_slice(p.as_bytes());430 +        stdin_bytes.push(0);431 +    }432 +    let a_check: &OsStr = OsStr::new("check-ignore");433 +    let a_no_index: &OsStr = OsStr::new("--no-index");434 +    let a_stdin: &OsStr = OsStr::new("--stdin");435 +    let a_z: &OsStr = OsStr::new("-z");436 +    let args: Vec<&OsStr> = vec![a_check, a_no_index, a_stdin, a_z];437 +    let out = run_git_stdin(repo, &args, stdin_bytes)?;438 +    if out.stdout.is_empty() {439 +        // Exit 0 (all tracked, none ignored) or 1 (no matches): same result.440 +        return Ok(HashSet::new());441 +    }442 +    if !out.status_ok {443 +        let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();444 +        return Err(GitError::Failed {445 +            cmd: cmd_string(repo, &args),446 +            status: out.status_str,447 +            stderr,448 +        });449 +    }450 +    let mut set = HashSet::new();451 +    for chunk in out.stdout.split(|b| *b == 0) {452 +        if chunk.is_empty() {453 +            continue;454 +        }455 +        set.insert(String::from_utf8_lossy(chunk).into_owned());456 +    }457 +    Ok(set)458 +}459 +460  /// Byte-exact file content for `branch:path`.461  ///462  /// `path` is a repo-relative path; non-UTF8 paths work because it is463 @@ -885,4 +1038,41 @@ mod tests {464          assert!(extract.join("zfile.txt").exists());465          assert!(extract.join("bin.dat").exists());466      }467 +468 +    #[test]469 +    fn ignored_paths_needs_no_index_for_tracked() {470 +        // hu13 lists tracked files, so force-added gitignored paths471 +        // must still be reported (plain check-ignore hides tracked).472 +        let dir = mktemp("ignored");473 +        git(&dir, &["init", "-b", "main"]);474 +        git(&dir, &["config", "user.name", "hu13-test"]);475 +        git(&dir, &["config", "user.email", "[email protected]"]);476 +        git(&dir, &["config", "commit.gpgsign", "false"]);477 +        std::fs::write(dir.join(".gitignore"), "ignored/\n*.log\n").unwrap();478 +        std::fs::create_dir_all(dir.join("ignored")).unwrap();479 +        std::fs::write(dir.join("ignored").join("secret.rs"), "fn s() {}\n").unwrap();480 +        std::fs::write(dir.join("debug.log"), "log\n").unwrap();481 +        std::fs::write(dir.join("src.rs"), "fn a() {}\n").unwrap();482 +        git(&dir, &["add", "-A"]);483 +        git(&dir, &["commit", "-m", "base"]);484 +        // Force-add the ignored paths so they are tracked like hu13 sees.485 +        git(&dir, &["add", "-f", "ignored/secret.rs", "debug.log"]);486 +        git(&dir, &["commit", "-m", "forced"]);487 +        let paths = vec![488 +            "src.rs".to_string(),489 +            "ignored/secret.rs".to_string(),490 +            "debug.log".to_string(),491 +            ".gitignore".to_string(),492 +        ];493 +        let ignored = ignored_paths(&dir, &paths).expect("ignored_paths");494 +        assert!(495 +            ignored.contains("ignored/secret.rs"),496 +            "force-added dir match, got {ignored:?}"497 +        );498 +        assert!(ignored.contains("debug.log"), "glob match, got {ignored:?}");499 +        assert!(!ignored.contains("src.rs"), "code kept, got {ignored:?}");500 +        assert!(!ignored.contains(".gitignore"), "gitignore kept");501 +        // Empty input never touches git.502 +        assert!(ignored_paths(&dir, &[]).expect("empty").is_empty());503 +    }504  }505 -- 506 2.55.0507