irongit
Sign inRegister
huncholanehuncholane/irongitPublic

Git hosting and a container registry in one Rust binary (axum + Astro)

Languages: skip minified JS/CSS like Linguist, add Slint and more languages, recompute cached stats

huncholanehuncholaneauthored
parent b15b911commit 9a22be439541cea57f8f59709c75c1141441452bBrowse files

2 files changed, +92 -2

+3-0backend/migrations/0004_languages_recount.sql
@@ -0,0 +1,3 @@
1+-- The language detector learned Linguist's minified rule and more languages;
2+-- drop the cached results so the startup backfill recomputes them.
3+delete from repo_languages;
+89-2backend/src/languages.rs
@@ -19,6 +19,20 @@ use crate::{git::Git, models::Repo, state::AppState};
1919 /// Languages that count, with GitHub's colors. Data and prose formats are
2020 /// deliberately absent: they never appear in the breakdown.
2121 const LANGUAGES: &[(&str, &str)] = &[
22+ ("V", "#4f87c4"),
23+ ("Typst", "#239dad"),
24+ ("Starlark", "#76d275"),
25+ ("Reason", "#ff5847"),
26+ ("ReScript", "#ed5051"),
27+ ("QML", "#44a51c"),
28+ ("Odin", "#60AFFE"),
29+ ("Mojo", "#ff4c1f"),
30+ ("Meson", "#007800"),
31+ ("Haxe", "#df7900"),
32+ ("Gleam", "#ffaff3"),
33+ ("Cairo", "#ff4a48"),
34+ ("Bicep", "#519aba"),
35+ ("Apex", "#1797c0"),
2236 ("Assembly", "#6E4C13"),
2337 ("Astro", "#ff5a03"),
2438 ("Awk", "#c30e9b"),
@@ -84,6 +98,7 @@ const LANGUAGES: &[(&str, &str)] = &[
8498 ("Scala", "#c22d40"),
8599 ("Scheme", "#1e4aec"),
86100 ("Shell", "#89e051"),
101+ ("Slint", "#2379F4"),
87102 ("Solidity", "#AA6746"),
88103 ("Svelte", "#ff3e00"),
89104 ("Swift", "#F05138"),
@@ -117,6 +132,8 @@ pub fn detect(path: &str) -> Option<&'static str> {
117132 "CMakeLists.txt" => Some("CMake"),
118133 "Rakefile" | "Gemfile" | "Vagrantfile" | "Podfile" | "Fastfile" => Some("Ruby"),
119134 "Jenkinsfile" => Some("Groovy"),
135+ "meson.build" => Some("Meson"),
136+ "BUILD" | "BUILD.bazel" | "WORKSPACE" => Some("Starlark"),
120137 "PKGBUILD" | ".bashrc" | ".bash_profile" | ".zshrc" | ".profile" => Some("Shell"),
121138 _ if name.starts_with("Dockerfile.") || name.ends_with(".Dockerfile") || name.ends_with(".dockerfile") => Some("Dockerfile"),
122139 _ => None,
@@ -202,6 +219,19 @@ pub fn detect(path: &str) -> Option<&'static str> {
202219 "tcl" => "Tcl",
203220 "awk" => "Awk",
204221 "pgsql" | "plpgsql" => "PLpgSQL",
222+ "slint" => "Slint",
223+ "bicep" => "Bicep",
224+ "cairo" => "Cairo",
225+ "gleam" => "Gleam",
226+ "hx" => "Haxe",
227+ "mojo" => "Mojo",
228+ "odin" => "Odin",
229+ "qml" => "QML",
230+ "res" | "resi" => "ReScript",
231+ "re" | "rei" => "Reason",
232+ "star" | "bzl" => "Starlark",
233+ "typ" => "Typst",
234+ "v" => "V",
205235 _ => return None,
206236 })
207237 }
@@ -280,6 +310,12 @@ impl Overrides {
280310 Self { rules }
281311 }
282312
313+ /// Whether any rule mentions this path (then the repo has decided and
314+ /// content heuristics stay out of it).
315+ fn decides(&self, path: &str) -> bool {
316+ self.rules.iter().any(|(set, _)| set.is_match(path))
317+ }
318+
283319 /// Later lines win, as in git.
284320 fn apply(&self, path: &str, excluded: bool, language: Option<&'static str>) -> (bool, Option<&'static str>) {
285321 let (mut excluded, mut language) = (excluded, language);
@@ -315,7 +351,8 @@ pub async fn compute(git: &Git, commit: &str) -> anyhow::Result<Vec<LanguageByte
315351 .output()
316352 .await?;
317353 anyhow::ensure!(output.status.success(), "git ls-tree failed");
318- let mut totals: std::collections::HashMap<&'static str, u64> = std::collections::HashMap::new();
354+ // First pass: which files count, and which JS/CSS need a minified check.
355+ let mut counted: Vec<(&'static str, u64, String)> = Vec::new();
319356 for record in output.stdout.split(|b| *b == 0).filter(|r| !r.is_empty()) {
320357 let record = String::from_utf8_lossy(record);
321358 let Some((meta, path)) = record.split_once('\t') else { continue };
@@ -326,19 +363,69 @@ pub async fn compute(git: &Git, commit: &str) -> anyhow::Result<Vec<LanguageByte
326363 continue;
327364 }
328365 let Ok(size) = size.parse::<u64>() else { continue };
366+ let Some(sha) = meta.split_whitespace().nth(2) else { continue };
329367 let (excluded, language) = overrides.apply(path, excluded_by_default(path), detect(path));
330368 if excluded {
331369 continue;
332370 }
333371 if let Some(language) = language {
334- *totals.entry(language).or_default() += size;
372+ let check = size > 1024 && matches!(language, "JavaScript" | "CSS") && !overrides.decides(path);
373+ counted.push((language, size, if check { sha.to_string() } else { String::new() }));
335374 }
336375 }
376+ let minified = minified_blobs(git, counted.iter().filter(|c| !c.2.is_empty()).map(|c| c.2.clone()).collect()).await.unwrap_or_default();
377+ let mut totals: std::collections::HashMap<&'static str, u64> = std::collections::HashMap::new();
378+ for (language, size, sha) in counted {
379+ if !sha.is_empty() && minified.contains(&sha) {
380+ continue;
381+ }
382+ *totals.entry(language).or_default() += size;
383+ }
337384 let mut out: Vec<LanguageBytes> = totals.into_iter().filter(|(_, b)| *b > 0).map(|(name, bytes)| LanguageBytes { name: name.into(), bytes }).collect();
338385 out.sort_by(|a, b| b.bytes.cmp(&a.bytes).then_with(|| a.name.cmp(&b.name)));
339386 Ok(out)
340387 }
341388
389+/// Linguist's minified rule: a JS or CSS file whose lines average more than
390+/// 110 characters is generated (a bundle or minified library) and does not
391+/// count. Reads the candidates' contents in one `git cat-file --batch`.
392+async fn minified_blobs(git: &Git, shas: Vec<String>) -> anyhow::Result<std::collections::HashSet<String>> {
393+ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
394+ let mut out = std::collections::HashSet::new();
395+ if shas.is_empty() {
396+ return Ok(out);
397+ }
398+ let mut child = git.command().args(["cat-file", "--batch"]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn()?;
399+ let mut stdin = child.stdin.take().ok_or_else(|| anyhow::anyhow!("stdin"))?;
400+ let wanted = shas.clone();
401+ let writer = tokio::spawn(async move {
402+ for sha in wanted {
403+ if stdin.write_all(format!("{sha}\n").as_bytes()).await.is_err() {
404+ break;
405+ }
406+ }
407+ });
408+ let mut reader = BufReader::new(child.stdout.take().ok_or_else(|| anyhow::anyhow!("stdout"))?);
409+ let mut header = String::new();
410+ for sha in &shas {
411+ header.clear();
412+ if reader.read_line(&mut header).await? == 0 {
413+ break;
414+ }
415+ let size: usize = header.split_whitespace().nth(2).and_then(|s| s.parse().ok()).unwrap_or(0);
416+ let mut content = vec![0u8; size + 1];
417+ reader.read_exact(&mut content).await?;
418+ let content = &content[..size];
419+ let lines = content.split(|b| *b == b'\n').filter(|l| !l.is_empty()).count().max(1);
420+ if content.len() / lines > 110 {
421+ out.insert(sha.clone());
422+ }
423+ }
424+ let _ = writer.await;
425+ let _ = child.wait().await;
426+ Ok(out)
427+}
428+
342429 /// The cached breakdown for the repo's default branch, recomputed when the
343430 /// branch has moved since. Also keeps `repos.primary_language` current.
344431 pub async fn for_repo(state: &AppState, repo: &Repo) -> anyhow::Result<Vec<LanguageBytes>> {