1 //! Build-time syntax highlighting (f0rge #73).2 //!3 //! No JS, no dependencies: a small line scanner that carries only block4 //! comment state between lines. Unknown languages return `None` and the5 //! file page stays plain. Output is escaped HTML with `tok-*` spans;6 //! every byte of input goes through [`esc`], never raw.7 //!8 //! Deliberately conservative: a mis-highlighted token must never change9 //! the text, only its color, and unterminated strings/comments degrade10 //! to "rest of line" rather than swallowing the file.11 12 use crate::layout::esc;13 14 /// Files larger than this stay plain: token spans would triple the page.15 pub const MAX_HIGHLIGHT_BYTES: usize = 512 * 1024;16 17 struct Lang {18 keywords: &'static [&'static str],19 line_comment: &'static [&'static str],20 block: Option<(&'static str, &'static str)>,21 strings: &'static [char],22 }23 24 const RUST: Lang = Lang {25 keywords: &[26 "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum",27 "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move",28 "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait",29 "true", "type", "unsafe", "use", "where", "while", "yield",30 ],31 line_comment: &["//"],32 block: Some(("/*", "*/")),33 strings: &['"'],34 };35 36 const JS: Lang = Lang {37 keywords: &[38 "async",39 "await",40 "break",41 "case",42 "catch",43 "class",44 "const",45 "continue",46 "default",47 "delete",48 "do",49 "else",50 "export",51 "extends",52 "false",53 "finally",54 "for",55 "from",56 "function",57 "if",58 "import",59 "in",60 "instanceof",61 "let",62 "new",63 "null",64 "of",65 "return",66 "static",67 "super",68 "switch",69 "this",70 "throw",71 "true",72 "try",73 "typeof",74 "undefined",75 "var",76 "void",77 "while",78 "yield",79 ],80 line_comment: &["//"],81 block: Some(("/*", "*/")),82 strings: &['"', '\'', '`'],83 };84 85 const JSON: Lang = Lang {86 keywords: &["true", "false", "null"],87 line_comment: &[],88 block: None,89 strings: &['"'],90 };91 92 const YAML: Lang = Lang {93 keywords: &["true", "false", "null", "yes", "no", "on", "off"],94 line_comment: &["#"],95 block: None,96 strings: &['"', '\''],97 };98 99 const TOML: Lang = Lang {100 keywords: &["true", "false"],101 line_comment: &["#"],102 block: None,103 strings: &['"', '\''],104 };105 106 const SHELL: Lang = Lang {107 keywords: &[108 "case", "cd", "do", "done", "echo", "elif", "else", "esac", "exec", "exit", "export", "fi",109 "for", "function", "if", "in", "local", "read", "return", "set", "shift", "then", "until",110 "unset", "while",111 ],112 line_comment: &["#"],113 block: None,114 strings: &['"', '\''],115 };116 117 const PYTHON: Lang = Lang {118 keywords: &[119 "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del",120 "elif", "else", "except", "False", "finally", "for", "from", "global", "if", "import",121 "in", "is", "lambda", "None", "nonlocal", "not", "or", "pass", "raise", "return", "True",122 "try", "while", "with", "yield",123 ],124 line_comment: &["#"],125 block: None,126 strings: &['"', '\''],127 };128 129 const GO: Lang = Lang {130 keywords: &[131 "break",132 "case",133 "chan",134 "const",135 "continue",136 "default",137 "defer",138 "else",139 "fallthrough",140 "for",141 "func",142 "go",143 "goto",144 "if",145 "import",146 "interface",147 "map",148 "package",149 "range",150 "return",151 "select",152 "struct",153 "switch",154 "type",155 "var",156 ],157 line_comment: &["//"],158 block: Some(("/*", "*/")),159 strings: &['"', '`'],160 };161 162 const CSS: Lang = Lang {163 keywords: &[164 "color",165 "background",166 "border",167 "display",168 "flex",169 "grid",170 "margin",171 "padding",172 "position",173 "width",174 "height",175 "font",176 "content",177 "media",178 "import",179 ],180 line_comment: &[],181 block: Some(("/*", "*/")),182 strings: &['"', '\''],183 };184 185 fn lang_for(path: &str) -> Option<&'static Lang> {186 let ext = path.rsplit('.').next()?.to_ascii_lowercase();187 match ext.as_str() {188 "rs" => Some(&RUST),189 "js" | "mjs" | "cjs" | "jsx" | "ts" | "tsx" => Some(&JS),190 "json" | "jsonl" => Some(&JSON),191 "yaml" | "yml" => Some(&YAML),192 "toml" | "ini" | "cfg" | "conf" => Some(&TOML),193 "sh" | "bash" | "zsh" | "fish" => Some(&SHELL),194 "py" => Some(&PYTHON),195 "go" => Some(&GO),196 "css" | "scss" | "less" => Some(&CSS),197 _ => None,198 }199 }200 201 fn is_ident_start(c: char) -> bool {202 c.is_alphabetic() || c == '_' || c == '$'203 }204 205 fn is_ident(c: char) -> bool {206 c.is_alphanumeric() || c == '_' || c == '$'207 }208 209 fn push_span(out: &mut String, class: &str, text: &str) {210 if text.is_empty() {211 return;212 }213 out.push_str("<span class=\"tok tok-");214 out.push_str(class);215 out.push_str("\">");216 out.push_str(&esc(text));217 out.push_str("</span>");218 }219 220 /// Highlight one file into one HTML fragment per input line.221 ///222 /// `None` for unknown languages, oversized input, or a path with no223 /// useful lexer (markdown, plain text) — callers then escape as-is.224 #[must_use]225 pub fn highlight_lines(path: &str, text: &str) -> Option<Vec<String>> {226 if text.len() > MAX_HIGHLIGHT_BYTES {227 return None;228 }229 let lang = lang_for(path)?;230 let mut in_block = false;231 let mut out = Vec::new();232 for line in text.split('\n') {233 out.push(scan_line(lang, line, &mut in_block));234 }235 Some(out)236 }237 238 fn starts_with_at(chars: &[char], at: usize, pat: &str) -> bool {239 for (i, c) in (at..).zip(pat.chars()) {240 if chars.get(i) != Some(&c) {241 return false;242 }243 }244 true245 }246 247 /// Find `pat` at or after char index `from`, without building a248 /// `String` for the rest of the line (the old scanner did that once per249 /// character, which made minified files quadratic; f0rge #78).250 fn find_at(chars: &[char], from: usize, pat: &str) -> Option<usize> {251 let len = pat.chars().count();252 if len == 0 {253 return Some(from);254 }255 let last = chars.len().checked_sub(len)?;256 if from > last {257 return None;258 }259 (from..=last).find(|&i| starts_with_at(chars, i, pat))260 }261 262 fn slice(chars: &[char], from: usize, to: usize) -> String {263 chars[from..to].iter().collect()264 }265 266 fn scan_line(lang: &Lang, line: &str, in_block: &mut bool) -> String {267 let chars: Vec<char> = line.chars().collect();268 let mut out = String::with_capacity(line.len() + 16);269 let mut i = 0;270 while i < chars.len() {271 if *in_block {272 if let Some((_, end)) = lang.block {273 if let Some(end_at) = find_at(&chars, i, end) {274 let stop = end_at + end.chars().count();275 push_span(&mut out, "com", &slice(&chars, i, stop));276 *in_block = false;277 i = stop;278 continue;279 }280 }281 push_span(&mut out, "com", &slice(&chars, i, chars.len()));282 return out;283 }284 285 if let Some((start, end)) = lang.block {286 if starts_with_at(&chars, i, start) {287 let after_start = i + start.chars().count();288 if let Some(end_at) = find_at(&chars, after_start, end) {289 let stop = end_at + end.chars().count();290 push_span(&mut out, "com", &slice(&chars, i, stop));291 i = stop;292 continue;293 }294 *in_block = true;295 push_span(&mut out, "com", &slice(&chars, i, chars.len()));296 return out;297 }298 }299 300 if lang301 .line_comment302 .iter()303 .any(|m| starts_with_at(&chars, i, m))304 {305 push_span(&mut out, "com", &slice(&chars, i, chars.len()));306 return out;307 }308 309 let c = chars[i];310 if lang.strings.contains(&c) {311 let mut j = i + 1;312 let mut escaped = false;313 while j < chars.len() {314 let d = chars[j];315 if escaped {316 escaped = false;317 } else if d == '\\' && c != '\'' {318 escaped = true;319 } else if d == c {320 j += 1;321 break;322 }323 j += 1;324 }325 push_span(&mut out, "str", &slice(&chars, i, j));326 i = j;327 continue;328 }329 330 if c.is_ascii_digit() {331 let mut j = i + 1;332 while j < chars.len()333 && (chars[j].is_ascii_alphanumeric() || matches!(chars[j], '.' | '_' | 'x' | 'X'))334 {335 j += 1;336 }337 push_span(&mut out, "num", &slice(&chars, i, j));338 i = j;339 continue;340 }341 342 if is_ident_start(c) {343 let mut j = i + 1;344 while j < chars.len() && is_ident(chars[j]) {345 j += 1;346 }347 let word = slice(&chars, i, j);348 let next_non_space = chars[j..].iter().find(|c| !c.is_whitespace());349 let call = next_non_space.is_some_and(|c| *c == '(');350 let class = if lang.keywords.contains(&word.as_str()) {351 Some("kw")352 } else if call {353 Some("fn")354 } else if word.chars().next().is_some_and(char::is_uppercase) {355 Some("type")356 } else {357 None358 };359 match class {360 Some(class) => push_span(&mut out, class, &word),361 None => out.push_str(&esc(&word)),362 }363 i = j;364 continue;365 }366 367 out.push_str(&esc(&c.to_string()));368 i += 1;369 }370 out371 }372 373 #[cfg(test)]374 mod tests {375 use super::*;376 377 #[test]378 fn rust_tokens_are_classified() {379 let src = "// hi\nfn main() { let x = 42; println!(\"ok\"); }";380 let lines = highlight_lines("a.rs", src).expect("rust");381 assert!(lines[0].contains("tok-com"));382 assert!(lines[1].contains("tok-kw\">fn<"));383 assert!(lines[1].contains("tok-fn\">main<"));384 assert!(lines[1].contains("tok-num\">42<"));385 assert!(lines[1].contains("tok-str\">"));386 // Text is preserved exactly once tags are stripped.387 let mut stripped = String::new();388 let mut in_tag = false;389 for c in lines.join("\n").chars() {390 match c {391 '<' => in_tag = true,392 '>' => in_tag = false,393 _ if !in_tag => stripped.push(c),394 _ => {}395 }396 }397 assert!(stripped.contains("fn main() { let x = 42;"));398 }399 400 #[test]401 fn unknown_languages_stay_plain() {402 assert!(highlight_lines("notes.txt", "fn main() {}").is_none());403 assert!(highlight_lines("README.md", "# hi").is_none());404 }405 406 #[test]407 fn block_comment_carries_across_lines() {408 let lines = highlight_lines("a.rs", "/* one\ntwo */\nfn x() {}").expect("rust");409 assert!(lines[0].contains("tok-com"));410 assert!(lines[1].contains("tok-com"));411 assert!(lines[2].contains("tok-kw"));412 }413 414 #[test]415 fn hostile_text_is_escaped() {416 let lines =417 highlight_lines("a.js", "const x = \"<script>alert(1)</script>\";").expect("js");418 assert!(!lines[0].contains("<script>"));419 assert!(lines[0].contains("<script>"));420 }421 422 #[test]423 fn long_single_line_is_not_quadratic() {424 // 64 KiB on one line took ~27s with the per-char suffix clone;425 // the linear scanner stays far below this coarse bound (f0rge #78).426 let big = "var x=1;".repeat(8000);427 let start = std::time::Instant::now();428 let lines = highlight_lines("min.js", &big).expect("js");429 assert!(start.elapsed() < std::time::Duration::from_secs(5));430 assert!(!lines[0].is_empty());431 }432 433 #[test]434 fn oversized_inputs_stay_plain() {435 let big = "x".repeat(MAX_HIGHLIGHT_BYTES + 1);436 assert!(highlight_lines("a.rs", &big).is_none());437 }438 }
/
•
•