Git hosting and a container registry in one Rust binary (axum + Astro)

Languages: break unresolved ties toward popular languages like Linguist's classifier prior; quiet known data-only heuristic patterns

huncholaneauthored 1 hour ago

parent e584fdfcommit 2364aeeb687f6a6d524ce1c727bde85e28693a67Browse files

2 files changed, +23 -6

+3-0backend/migrations/0007_languages_popular.sql

		@@ -0,0 +1,3 @@
	1	+-- Ambiguous files with no matching heuristic now prefer the more popular
	2	+-- language, like Linguist's classifier prior; recount.
	3	+delete from repo_languages;

+20-6backend/src/languages/linguist.rs

		@@ -6,8 +6,9 @@
6	6	//! Detection runs Linguist's strategies in its order: modeline, filename,
7	7	//! shebang, extension, XML, heuristics. Each narrows the candidates and the
8	8	//! first to leave exactly one decides. Where Linguist would finally ask its
9		-//! Bayesian classifier, the candidate whose primary extension matches wins,
10		-//! then the first by name.
	9	+//! Bayesian classifier, whose prior favours languages with more samples, the
	10	+//! more popular candidate wins (Linguist once shipped this as popular.yml),
	11	+//! then the one whose primary extension matches.
11	12
12	13	use std::{
13	14	collections::HashMap,
		@@ -114,7 +115,8 @@ impl Pattern {
114	115	.get_or_init(\|\| match regex::bytes::RegexBuilder::new(&self.source).size_limit(64 << 20).build() {
115	116	Ok(regex) => Some(regex),
116	117	Err(error) => {
117		- tracing::warn!(pattern = %self.source, %error, "linguist heuristic does not compile");
	118	+ // Only data/prose rules use syntax Rust lacks (a test checks).
	119	+ tracing::debug!(pattern = %self.source, %error, "linguist heuristic does not compile");
118	120	None
119	121	}
120	122	})
		@@ -387,12 +389,14 @@ impl Linguist {
387	389	}
388	390	}
389	391	}
390		- // Linguist's classifier would decide here; prefer the language this
391		- // extension belongs to first.
	392	+ // Linguist's classifier would decide here.
392	393	let extension = extensions(&basename(path).to_lowercase()).into_iter().last();
	394	+ let popular = \|l: &Lang\| POPULAR.iter().position(\|p\| p == self.name(l)).unwrap_or(usize::MAX);
393	395	candidates
394	396	.iter()
395		- .find(\|l\| extension.is_some() && self.language(**l).primary_extension == extension)
	397	+ .filter(\|l\| popular(l) != usize::MAX)
	398	+ .min_by_key(\|l\| popular(l))
	399	+ .or_else(\|\| candidates.iter().find(\|l\| extension.is_some() && self.language(**l).primary_extension == extension))
396	400	.or(candidates.first())
397	401	.copied()
398	402	}
		@@ -404,6 +408,13 @@ impl Linguist {
404	408	}
405	409	}
406	410
	411	+/// The tie-breaker for candidates no rule separates, most popular first.
	412	+const POPULAR: &[&str] = &[
	413	+ "JavaScript", "Python", "TypeScript", "Java", "C#", "C++", "C", "PHP", "Shell", "Go", "Ruby", "Rust", "Kotlin", "Swift", "HTML",
	414	+ "CSS", "Markdown", "JSON", "YAML", "SQL", "Dart", "Lua", "Perl", "R", "Scala", "Objective-C", "PowerShell", "Haskell", "Elixir",
	415	+ "Clojure", "Groovy", "Vim script", "Makefile", "Dockerfile", "XML", "Text",
	416	+];
	417	+
407	418	fn narrow(candidates: &mut Vec<Lang>, found: Vec<Lang>) -> Option<Lang> {
408	419	let found: Vec<Lang> = if candidates.is_empty() { found } else { candidates.iter().filter(\|c\| found.contains(c)).copied().collect() };
409	420	match found.len() {
		@@ -745,6 +756,9 @@ mod tests {
745	756	assert_eq!(detect("db/0002.sql", Some("create table t (id int);\n")).as_deref(), Some("SQL"));
746	757	assert_eq!(detect("script.txt", Some("# vim: set ft=python:\nprint(1)\n")).as_deref(), Some("Python"));
747	758	assert_eq!(detect("README.md", None).as_deref(), Some("Markdown"));
	759	+ // No rule fits; the classifier's prior favours the popular language.
	760	+ assert_eq!(detect("regex/Event.re", Some("# a comment\n^(?<start>\\d+)\n")).as_deref(), Some("C++"));
	761	+ assert_eq!(detect("lib/x.re", Some("open Belt;\nlet x = 1;\n")).as_deref(), Some("Reason"));
748	762	}
749	763
750	764	#[test]