irongit
Sign inRegister
huncholanehuncholane/irongitPublic

Git hosting and a container registry in one Rust binary (axum + Astro)

Languages: skip generated files (Go DO NOT EDIT headers, protoc, Thrift, Cython, @generated) like Linguist, recompute cached stats

huncholanehuncholaneauthored
parent 9a22be4commit 426b72b26846ed24675b288a798a7e19ee34e66cBrowse files

2 files changed, +59 -12

+3-0backend/migrations/0005_languages_generated.sql
@@ -0,0 +1,3 @@
1+-- Generated files (Go "Code generated ... DO NOT EDIT", protoc, Thrift,
2+-- Cython, @generated) no longer count; recompute every cached breakdown.
3+delete from repo_languages;
+56-12backend/src/languages.rs
@@ -369,14 +369,29 @@ pub async fn compute(git: &Git, commit: &str) -> anyhow::Result<Vec<LanguageByte
369369 continue;
370370 }
371371 if let Some(language) = language {
372- let check = size > 1024 && matches!(language, "JavaScript" | "CSS") && !overrides.decides(path);
372+ // Content is only checked where the repo has not said otherwise.
373+ let check = size <= CONTENT_CHECK_MAX_FILE && !overrides.decides(path);
373374 counted.push((language, size, if check { sha.to_string() } else { String::new() }));
374375 }
375376 }
376- let minified = minified_blobs(git, counted.iter().filter(|c| !c.2.is_empty()).map(|c| c.2.clone()).collect()).await.unwrap_or_default();
377+ // Read file contents within a budget; past it, keep to minified JS/CSS.
378+ let mut budget = CONTENT_CHECK_BUDGET;
379+ let candidates: Vec<(String, bool)> = counted
380+ .iter()
381+ .filter(|(_, _, sha)| !sha.is_empty())
382+ .filter_map(|(language, size, sha)| {
383+ let minify_check = matches!(*language, "JavaScript" | "CSS") && *size > 1024;
384+ let wanted = budget >= *size || minify_check;
385+ if wanted {
386+ budget = budget.saturating_sub(*size);
387+ }
388+ wanted.then(|| (sha.clone(), minify_check))
389+ })
390+ .collect();
391+ let generated = generated_blobs(git, candidates).await.unwrap_or_default();
377392 let mut totals: std::collections::HashMap<&'static str, u64> = std::collections::HashMap::new();
378393 for (language, size, sha) in counted {
379- if !sha.is_empty() && minified.contains(&sha) {
394+ if !sha.is_empty() && generated.contains(&sha) {
380395 continue;
381396 }
382397 *totals.entry(language).or_default() += size;
@@ -386,18 +401,37 @@ pub async fn compute(git: &Git, commit: &str) -> anyhow::Result<Vec<LanguageByte
386401 Ok(out)
387402 }
388403
389-/// Linguist's minified rule: a JS or CSS file whose lines average more than
390-/// 110 characters is generated (a bundle or minified library) and does not
391-/// count. Reads the candidates' contents in one `git cat-file --batch`.
392-async fn minified_blobs(git: &Git, shas: Vec<String>) -> anyhow::Result<std::collections::HashSet<String>> {
404+/// Files larger than this are never read for the content checks.
405+const CONTENT_CHECK_MAX_FILE: u64 = 8 * 1024 * 1024;
406+/// Total bytes read per computation before checks narrow to JS/CSS.
407+const CONTENT_CHECK_BUDGET: u64 = 64 * 1024 * 1024;
408+
409+/// A generated-file marker in the first lines, as Linguist recognises them:
410+/// Go's "Code generated ... DO NOT EDIT." convention and the headers protoc,
411+/// Thrift, Cython and Relay-style `@generated` tools write.
412+fn has_generated_header(content: &[u8]) -> bool {
413+ let head = String::from_utf8_lossy(&content[..content.len().min(2048)]).to_ascii_lowercase();
414+ head.lines().take(10).any(|line| {
415+ (line.contains("code generated") && line.contains("do not edit"))
416+ || line.contains("@generated")
417+ || line.contains("generated by the protocol buffer compiler")
418+ || line.contains("autogenerated by thrift")
419+ || line.contains("generated by cython")
420+ })
421+}
422+
423+/// Linguist's content rules: a file with a generated header, or a JS/CSS file
424+/// whose lines average more than 110 characters (a bundle or minified
425+/// library), does not count. Reads the candidates in one `cat-file --batch`.
426+async fn generated_blobs(git: &Git, blobs: Vec<(String, bool)>) -> anyhow::Result<std::collections::HashSet<String>> {
393427 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
394428 let mut out = std::collections::HashSet::new();
395- if shas.is_empty() {
429+ if blobs.is_empty() {
396430 return Ok(out);
397431 }
398432 let mut child = git.command().args(["cat-file", "--batch"]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn()?;
399433 let mut stdin = child.stdin.take().ok_or_else(|| anyhow::anyhow!("stdin"))?;
400- let wanted = shas.clone();
434+ let wanted: Vec<String> = blobs.iter().map(|(sha, _)| sha.clone()).collect();
401435 let writer = tokio::spawn(async move {
402436 for sha in wanted {
403437 if stdin.write_all(format!("{sha}\n").as_bytes()).await.is_err() {
@@ -407,17 +441,19 @@ async fn minified_blobs(git: &Git, shas: Vec<String>) -> anyhow::Result<std::col
407441 });
408442 let mut reader = BufReader::new(child.stdout.take().ok_or_else(|| anyhow::anyhow!("stdout"))?);
409443 let mut header = String::new();
410- for sha in &shas {
444+ for (sha, minify_check) in &blobs {
411445 header.clear();
412446 if reader.read_line(&mut header).await? == 0 {
413447 break;
414448 }
415- let size: usize = header.split_whitespace().nth(2).and_then(|s| s.parse().ok()).unwrap_or(0);
449+ // "<sha> missing" carries no body to read past.
450+ let Some(size) = header.split_whitespace().nth(2).and_then(|s| s.parse::<usize>().ok()) else { continue };
416451 let mut content = vec![0u8; size + 1];
417452 reader.read_exact(&mut content).await?;
418453 let content = &content[..size];
419454 let lines = content.split(|b| *b == b'\n').filter(|l| !l.is_empty()).count().max(1);
420- if content.len() / lines > 110 {
455+ let minified = *minify_check && content.len() / lines > 110;
456+ if minified || has_generated_header(content) {
421457 out.insert(sha.clone());
422458 }
423459 }
@@ -543,6 +579,14 @@ mod tests {
543579 assert_eq!(overrides.apply("lib/a.inc", false, None), (false, Some("PHP")));
544580 }
545581
582+ #[test]
583+ fn generated_headers() {
584+ assert!(has_generated_header(b"// Code generated by gen_sort_variants.go; DO NOT EDIT.\n\npackage x"));
585+ assert!(has_generated_header(b"// @generated by relay-compiler\n"));
586+ assert!(has_generated_header(b"# Generated by the protocol buffer compiler. DO NOT EDIT!\n"));
587+ assert!(!has_generated_header(b"// Package sort implements sorting.\npackage sort"));
588+ }
589+
546590 #[test]
547591 fn small_languages_fold_into_other() {
548592 let stats = vec![