Git hosting and a container registry in one Rust binary (axum + Astro)

Languages: Linguist-style breakdown per repo (bar and legend), primary language in lists, API and CLI; profile spider chart on a log scale

huncholaneauthored 53 minutes ago

parent 69a304bcommit b15b9114d08217953b8c72f41ff852a7c507a853Browse files

16 files changed, +667 -3

+24-0Cargo.lock

		@@ -440,6 +440,16 @@ dependencies = [
440	440	"syn 3.0.6",
441	441	]
442	442
	443	+[[package]]
	444	+name = "bstr"
	445	+version = "1.13.1"
	446	+source = "registry+https://github.com/rust-lang/crates.io-index"
	447	+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
	448	+dependencies = [
	449	+ "memchr",
	450	+ "serde_core",
	451	+]
	452	+
443	453	[[package]]
444	454	name = "bumpalo"
445	455	version = "3.20.3"
		@@ -1492,6 +1502,19 @@ dependencies = [
1492	1502	"zeroize",
1493	1503	]
1494	1504
	1505	+[[package]]
	1506	+name = "globset"
	1507	+version = "0.4.20"
	1508	+source = "registry+https://github.com/rust-lang/crates.io-index"
	1509	+checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
	1510	+dependencies = [
	1511	+ "aho-corasick",
	1512	+ "bstr",
	1513	+ "log",
	1514	+ "regex-automata",
	1515	+ "regex-syntax",
	1516	+]
	1517	+
1495	1518	[[package]]
1496	1519	name = "group"
1497	1520	version = "0.14.0"
		@@ -1974,6 +1997,7 @@ dependencies = [
1974	1997	"dotenvy",
1975	1998	"flate2",
1976	1999	"futures",
	2000	+ "globset",
1977	2001	"hex",
1978	2002	"hmac",
1979	2003	"irongit-shared",

+1-0backend/Cargo.toml

		@@ -51,3 +51,4 @@ clap = { version = "4.6.7", features = ["derive", "env"] }
51	51	lettre = { version = "0.11.23", default-features = false, features = ["tokio1-rustls-tls", "smtp-transport", "builder", "hostname"] }
52	52	axum-extra = { version = "0.12.6", features = ["cookie", "typed-header"] }
53	53	async-compression = { version = "0.4.50", features = ["tokio", "gzip"] }
	54	+globset = "0.4.20"

+12-0backend/migrations/0003_languages.sql

		@@ -0,0 +1,12 @@
	1	+-- Language breakdown per repository (bytes per language on the default
	2	+-- branch, Linguist-style), cached for the commit it was computed at.
	3	+
	4	+alter table repos add column primary_language text;
	5	+
	6	+create table repo_languages (
	7	+ repo_id bigint primary key references repos (id) on delete cascade,
	8	+ commit_sha text not null,
	9	+ -- [{"name": "Rust", "bytes": 12345}, ...], largest first
	10	+ stats jsonb not null,
	11	+ computed_at timestamptz not null default now()
	12	+);

+1-0backend/src/api/mod.rs

		@@ -177,6 +177,7 @@ pub fn repo_dto(config: &Config, repo: &Repo, permission: Option<Access>) -> sha
177	177	description: repo.description.clone(),
178	178	visibility: repo.visibility.clone(),
179	179	default_branch: repo.default_branch.clone(),
	180	+ language: repo.primary_language.clone(),
180	181	size_bytes: repo.size_bytes,
181	182	is_empty: repo.is_empty,
182	183	archived: repo.archived,

+5-0backend/src/importer/mod.rs

		@@ -564,6 +564,11 @@ async fn fill_repo(state: &AppState, gh: &GitHub, import: &ImportRow, repo: &Rep
564	564	return Err(anyhow!("took {} and put {} over its storage quota", crate::web::ui::bytes(size), owner.name));
565	565	}
566	566
	567	+ if let Some(fresh) = Repo::by_id(&state.db, repo.id).await? {
	568	+ if let Err(error) = crate::languages::for_repo(state, &fresh).await {
	569	+ tracing::warn!(repo = %fresh.full_name(), ?error, "language stats failed");
	570	+ }
	571	+ }
567	572	let note = (lfs_missing > 0).then(\|\| format!("{lfs_missing} Git LFS object(s) were missing on GitHub and were not copied"));
568	573	Ok(RepoOutcome { repo_id: Some(repo.id), commits, lfs, skipped: false, note })
569	574	}

+1-0backend/src/jobs.rs

		@@ -9,6 +9,7 @@ pub fn spawn_all(state: AppState) {
9	9	let recover_state = state.clone();
10	10	tokio::spawn(async move { crate::importer::recover(&recover_state).await });
11	11	crate::registry::spawn_gc(state.clone());
	12	+ crate::languages::spawn_backfill(state.clone());
12	13	crate::backup::spawn(state.clone());
13	14	tokio::spawn(prune_sessions(state));
14	15	}

+471-0backend/src/languages.rs

		@@ -0,0 +1,471 @@
	1	+//! Language breakdown of a repository, the way GitHub's Linguist does it:
	2	+//! bytes per language across the default branch, leaving out vendored,
	3	+//! generated and documentation files and data or prose formats (JSON, YAML,
	4	+//! Markdown). A repo's `.gitattributes` can override any of that with
	5	+//! `linguist-vendored`, `linguist-generated`, `linguist-documentation` and
	6	+//! `linguist-language=<Name>`.
	7	+//!
	8	+//! Results are cached per commit in `repo_languages`, so a page view costs a
	9	+//! lookup; recomputing walks the tree with `git ls-tree -r -l` (sizes only,
	10	+//! no file contents).
	11	+
	12	+use std::process::Stdio;
	13	+
	14	+use globset::{Glob, GlobSet, GlobSetBuilder};
	15	+use serde::{Deserialize, Serialize};
	16	+
	17	+use crate::{git::Git, models::Repo, state::AppState};
	18	+
	19	+/// Languages that count, with GitHub's colors. Data and prose formats are
	20	+/// deliberately absent: they never appear in the breakdown.
	21	+const LANGUAGES: &[(&str, &str)] = &[
	22	+ ("Assembly", "#6E4C13"),
	23	+ ("Astro", "#ff5a03"),
	24	+ ("Awk", "#c30e9b"),
	25	+ ("Batchfile", "#C1F12E"),
	26	+ ("C", "#555555"),
	27	+ ("C#", "#178600"),
	28	+ ("C++", "#f34b7d"),
	29	+ ("CMake", "#DA3434"),
	30	+ ("CSS", "#663399"),
	31	+ ("Clojure", "#db5855"),
	32	+ ("CoffeeScript", "#244776"),
	33	+ ("Common Lisp", "#3fb68b"),
	34	+ ("Crystal", "#000100"),
	35	+ ("Cuda", "#3A4E3A"),
	36	+ ("D", "#ba595e"),
	37	+ ("Dart", "#00B4AB"),
	38	+ ("Dockerfile", "#384d54"),
	39	+ ("EJS", "#a91e50"),
	40	+ ("Elixir", "#6e4a7e"),
	41	+ ("Elm", "#60B5CC"),
	42	+ ("Emacs Lisp", "#c065db"),
	43	+ ("Erlang", "#B83998"),
	44	+ ("F#", "#b845fc"),
	45	+ ("Fortran", "#4d41b1"),
	46	+ ("GDScript", "#355570"),
	47	+ ("GLSL", "#5686a5"),
	48	+ ("Go", "#00ADD8"),
	49	+ ("Groovy", "#4298b8"),
	50	+ ("HCL", "#844FBA"),
	51	+ ("HLSL", "#aace60"),
	52	+ ("HTML", "#e34c26"),
	53	+ ("Handlebars", "#f7931e"),
	54	+ ("Haskell", "#5e5086"),
	55	+ ("Java", "#b07219"),
	56	+ ("JavaScript", "#f1e05a"),
	57	+ ("Julia", "#a270ba"),
	58	+ ("Jupyter Notebook", "#DA5B0B"),
	59	+ ("Kotlin", "#A97BFF"),
	60	+ ("Less", "#1d365d"),
	61	+ ("Liquid", "#67b8de"),
	62	+ ("Lua", "#000080"),
	63	+ ("MDX", "#fcb32c"),
	64	+ ("Makefile", "#427819"),
	65	+ ("Nim", "#ffc200"),
	66	+ ("Nix", "#7e7eff"),
	67	+ ("OCaml", "#ef7a08"),
	68	+ ("Objective-C", "#438eff"),
	69	+ ("Objective-C++", "#6866fb"),
	70	+ ("PHP", "#4F5D95"),
	71	+ ("PLpgSQL", "#336790"),
	72	+ ("Pascal", "#E3F171"),
	73	+ ("Perl", "#0298c3"),
	74	+ ("PowerShell", "#012456"),
	75	+ ("Pug", "#a86454"),
	76	+ ("PureScript", "#1D222D"),
	77	+ ("Python", "#3572A5"),
	78	+ ("R", "#198CE7"),
	79	+ ("Racket", "#3c5caa"),
	80	+ ("Ruby", "#701516"),
	81	+ ("Rust", "#dea584"),
	82	+ ("SCSS", "#c6538c"),
	83	+ ("Sass", "#a53b70"),
	84	+ ("Scala", "#c22d40"),
	85	+ ("Scheme", "#1e4aec"),
	86	+ ("Shell", "#89e051"),
	87	+ ("Solidity", "#AA6746"),
	88	+ ("Svelte", "#ff3e00"),
	89	+ ("Swift", "#F05138"),
	90	+ ("Tcl", "#e4cc98"),
	91	+ ("TeX", "#3D6117"),
	92	+ ("Twig", "#c1d026"),
	93	+ ("TypeScript", "#3178c6"),
	94	+ ("Vala", "#a56de2"),
	95	+ ("Vim Script", "#199f4b"),
	96	+ ("Visual Basic .NET", "#945db7"),
	97	+ ("Vue", "#41b883"),
	98	+ ("WGSL", "#1a5e9a"),
	99	+ ("Zig", "#ec915c"),
	100	+];
	101	+
	102	+/// GitHub's color for a language, or a neutral grey for unknown names.
	103	+pub fn color(name: &str) -> &'static str {
	104	+ LANGUAGES.iter().find(\|(n, _)\| n == name).map(\|(_, c)\| c).unwrap_or("#8b949e")
	105	+}
	106	+
	107	+fn known(name: &str) -> Option<&'static str> {
	108	+ LANGUAGES.iter().find(\|(n, _)\| n.eq_ignore_ascii_case(name)).map(\|(n, _)\| *n)
	109	+}
	110	+
	111	+/// Language of a file by its name, then its extension.
	112	+pub fn detect(path: &str) -> Option<&'static str> {
	113	+ let name = path.rsplit('/').next().unwrap_or(path);
	114	+ let by_name = match name {
	115	+ "Dockerfile" \| "Containerfile" => Some("Dockerfile"),
	116	+ "Makefile" \| "makefile" \| "GNUmakefile" => Some("Makefile"),
	117	+ "CMakeLists.txt" => Some("CMake"),
	118	+ "Rakefile" \| "Gemfile" \| "Vagrantfile" \| "Podfile" \| "Fastfile" => Some("Ruby"),
	119	+ "Jenkinsfile" => Some("Groovy"),
	120	+ "PKGBUILD" \| ".bashrc" \| ".bash_profile" \| ".zshrc" \| ".profile" => Some("Shell"),
	121	+ _ if name.starts_with("Dockerfile.") \|\| name.ends_with(".Dockerfile") \|\| name.ends_with(".dockerfile") => Some("Dockerfile"),
	122	+ _ => None,
	123	+ };
	124	+ if by_name.is_some() {
	125	+ return by_name;
	126	+ }
	127	+ let extension = name.rsplit_once('.').map(\|(_, e)\| e.to_ascii_lowercase())?;
	128	+ Some(match extension.as_str() {
	129	+ "rs" => "Rust",
	130	+ "ts" \| "tsx" \| "mts" \| "cts" => "TypeScript",
	131	+ "js" \| "jsx" \| "mjs" \| "cjs" => "JavaScript",
	132	+ "py" \| "pyw" \| "pyi" => "Python",
	133	+ "go" => "Go",
	134	+ "java" => "Java",
	135	+ "kt" \| "kts" => "Kotlin",
	136	+ "swift" => "Swift",
	137	+ "c" \| "h" => "C",
	138	+ "cc" \| "cpp" \| "cxx" \| "c++" \| "hpp" \| "hh" \| "hxx" \| "ino" => "C++",
	139	+ "cs" => "C#",
	140	+ "m" => "Objective-C",
	141	+ "mm" => "Objective-C++",
	142	+ "rb" \| "rake" \| "gemspec" => "Ruby",
	143	+ "php" => "PHP",
	144	+ "html" \| "htm" \| "xhtml" => "HTML",
	145	+ "css" => "CSS",
	146	+ "scss" => "SCSS",
	147	+ "sass" => "Sass",
	148	+ "less" => "Less",
	149	+ "sh" \| "bash" \| "zsh" \| "ksh" \| "fish" => "Shell",
	150	+ "ps1" \| "psm1" => "PowerShell",
	151	+ "bat" \| "cmd" => "Batchfile",
	152	+ "lua" => "Lua",
	153	+ "dart" => "Dart",
	154	+ "vue" => "Vue",
	155	+ "svelte" => "Svelte",
	156	+ "astro" => "Astro",
	157	+ "ex" \| "exs" => "Elixir",
	158	+ "erl" \| "hrl" => "Erlang",
	159	+ "hs" \| "lhs" => "Haskell",
	160	+ "scala" \| "sc" => "Scala",
	161	+ "clj" \| "cljs" \| "cljc" \| "edn" => "Clojure",
	162	+ "ml" \| "mli" => "OCaml",
	163	+ "fs" \| "fsi" \| "fsx" => "F#",
	164	+ "zig" => "Zig",
	165	+ "nim" => "Nim",
	166	+ "jl" => "Julia",
	167	+ "r" => "R",
	168	+ "pl" \| "pm" => "Perl",
	169	+ "cmake" => "CMake",
	170	+ "nix" => "Nix",
	171	+ "tf" \| "hcl" => "HCL",
	172	+ "vim" => "Vim Script",
	173	+ "el" => "Emacs Lisp",
	174	+ "tex" \| "sty" \| "cls" => "TeX",
	175	+ "ipynb" => "Jupyter Notebook",
	176	+ "sol" => "Solidity",
	177	+ "glsl" \| "vert" \| "frag" => "GLSL",
	178	+ "wgsl" => "WGSL",
	179	+ "hlsl" => "HLSL",
	180	+ "cu" \| "cuh" => "Cuda",
	181	+ "asm" \| "s" \| "nasm" => "Assembly",
	182	+ "groovy" \| "gradle" => "Groovy",
	183	+ "cr" => "Crystal",
	184	+ "elm" => "Elm",
	185	+ "coffee" => "CoffeeScript",
	186	+ "mdx" => "MDX",
	187	+ "hbs" \| "handlebars" => "Handlebars",
	188	+ "pug" \| "jade" => "Pug",
	189	+ "twig" => "Twig",
	190	+ "liquid" => "Liquid",
	191	+ "ejs" => "EJS",
	192	+ "purs" => "PureScript",
	193	+ "rkt" => "Racket",
	194	+ "scm" \| "ss" => "Scheme",
	195	+ "lisp" \| "lsp" \| "cl" => "Common Lisp",
	196	+ "d" => "D",
	197	+ "f" \| "f90" \| "f95" \| "for" => "Fortran",
	198	+ "pas" \| "pp" => "Pascal",
	199	+ "vb" => "Visual Basic .NET",
	200	+ "vala" => "Vala",
	201	+ "gd" => "GDScript",
	202	+ "tcl" => "Tcl",
	203	+ "awk" => "Awk",
	204	+ "pgsql" \| "plpgsql" => "PLpgSQL",
	205	+ _ => return None,
	206	+ })
	207	+}
	208	+
	209	+/// Linguist's vendored, generated and documentation paths (the common ones).
	210	+fn excluded_by_default(path: &str) -> bool {
	211	+ let lower = path.to_ascii_lowercase();
	212	+ let segments: Vec<&str> = lower.split('/').collect();
	213	+ let dirs = &segments[..segments.len().saturating_sub(1)];
	214	+ let name = segments.last().copied().unwrap_or("");
	215	+ const VENDOR_DIRS: &[&str] = &[
	216	+ "node_modules", "vendor", "vendors", "third_party", "third-party", "thirdparty", "bower_components", "jspm_packages",
	217	+ "dist", "target", "__pycache__", ".venv", "venv", "site-packages", ".yarn", ".pnpm-store", "godeps", "pods", "carthage",
	218	+ ".git", ".github", "external", "deps",
	219	+ ];
	220	+ if dirs.iter().any(\|d\| VENDOR_DIRS.contains(d)) {
	221	+ return true;
	222	+ }
	223	+ // Documentation at the root, as Linguist treats it.
	224	+ if matches!(dirs.first(), Some(&"doc") \| Some(&"docs") \| Some(&"documentation") \| Some(&"examples")) {
	225	+ return true;
	226	+ }
	227	+ name.contains(".min.") \|\| name.ends_with("-min.js") \|\| name.ends_with(".bundle.js") \|\| name.ends_with(".pb.go") \|\| name.ends_with("_pb2.py")
	228	+ \|\| name.contains(".generated.") \|\| name.ends_with(".d.ts") \|\| name == "package-lock.json"
	229	+}
	230	+
	231	+/// Path overrides from the root `.gitattributes`.
	232	+#[derive(Default)]
	233	+struct Overrides {
	234	+ rules: Vec<(GlobSet, Rule)>,
	235	+}
	236	+
	237	+#[derive(Clone)]
	238	+enum Rule {
	239	+ Exclude(bool),
	240	+ Language(&'static str),
	241	+}
	242	+
	243	+impl Overrides {
	244	+ fn parse(text: &str) -> Self {
	245	+ let mut rules = Vec::new();
	246	+ for line in text.lines() {
	247	+ let line = line.trim();
	248	+ if line.is_empty() \|\| line.starts_with('#') {
	249	+ continue;
	250	+ }
	251	+ let mut parts = line.split_whitespace();
	252	+ let Some(pattern) = parts.next() else { continue };
	253	+ let glob = if let Some(rooted) = pattern.strip_prefix('/') {
	254	+ rooted.to_string()
	255	+ } else if pattern.contains('/') {
	256	+ pattern.to_string()
	257	+ } else {
	258	+ format!("**/{pattern}")
	259	+ };
	260	+ let glob = if glob.ends_with('/') { format!("{glob}**") } else { glob };
	261	+ let Ok(set) = Glob::new(&glob).and_then(\|g\| GlobSetBuilder::new().add(g).build()) else { continue };
	262	+ for attribute in parts {
	263	+ let (negated, attribute) = match attribute.strip_prefix('-') {
	264	+ Some(rest) => (true, rest),
	265	+ None => (false, attribute),
	266	+ };
	267	+ let (key, value) = attribute.split_once('=').unwrap_or((attribute, "true"));
	268	+ let enabled = !negated && value != "false";
	269	+ let rule = match key {
	270	+ "linguist-vendored" \| "linguist-generated" \| "linguist-documentation" => Rule::Exclude(enabled),
	271	+ "linguist-language" => match known(&value.replace('-', " ")) {
	272	+ Some(language) => Rule::Language(language),
	273	+ None => continue,
	274	+ },
	275	+ _ => continue,
	276	+ };
	277	+ rules.push((set.clone(), rule));
	278	+ }
	279	+ }
	280	+ Self { rules }
	281	+ }
	282	+
	283	+ /// Later lines win, as in git.
	284	+ fn apply(&self, path: &str, excluded: bool, language: Option<&'static str>) -> (bool, Option<&'static str>) {
	285	+ let (mut excluded, mut language) = (excluded, language);
	286	+ for (set, rule) in &self.rules {
	287	+ if set.is_match(path) {
	288	+ match rule {
	289	+ Rule::Exclude(on) => excluded = *on,
	290	+ Rule::Language(name) => language = Some(name),
	291	+ }
	292	+ }
	293	+ }
	294	+ (excluded, language)
	295	+ }
	296	+}
	297	+
	298	+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
	299	+pub struct LanguageBytes {
	300	+ pub name: String,
	301	+ pub bytes: u64,
	302	+}
	303	+
	304	+/// Bytes per language in `commit`, largest first.
	305	+pub async fn compute(git: &Git, commit: &str) -> anyhow::Result<Vec<LanguageBytes>> {
	306	+ let overrides = match git.object_at(commit, ".gitattributes").await? {
	307	+ Some((kind, sha, size)) if kind == "blob" && size < 256 * 1024 => Overrides::parse(&String::from_utf8_lossy(&git.read_blob(&sha).await?)),
	308	+ _ => Overrides::default(),
	309	+ };
	310	+ let output = git
	311	+ .command()
	312	+ .args(["ls-tree", "-r", "-l", "-z", "--end-of-options", commit])
	313	+ .stdout(Stdio::piped())
	314	+ .stderr(Stdio::null())
	315	+ .output()
	316	+ .await?;
	317	+ anyhow::ensure!(output.status.success(), "git ls-tree failed");
	318	+ let mut totals: std::collections::HashMap<&'static str, u64> = std::collections::HashMap::new();
	319	+ for record in output.stdout.split(\|b\| *b == 0).filter(\|r\| !r.is_empty()) {
	320	+ let record = String::from_utf8_lossy(record);
	321	+ let Some((meta, path)) = record.split_once('\t') else { continue };
	322	+ let mut fields = meta.split_whitespace();
	323	+ let (Some(mode), Some(kind), _, Some(size)) = (fields.next(), fields.next(), fields.next(), fields.next()) else { continue };
	324	+ // Blobs only: no submodules, no symlinks.
	325	+ if kind != "blob" \|\| mode == "120000" {
	326	+ continue;
	327	+ }
	328	+ let Ok(size) = size.parse::<u64>() else { continue };
	329	+ let (excluded, language) = overrides.apply(path, excluded_by_default(path), detect(path));
	330	+ if excluded {
	331	+ continue;
	332	+ }
	333	+ if let Some(language) = language {
	334	+ *totals.entry(language).or_default() += size;
	335	+ }
	336	+ }
	337	+ let mut out: Vec<LanguageBytes> = totals.into_iter().filter(\|(_, b)\| *b > 0).map(\|(name, bytes)\| LanguageBytes { name: name.into(), bytes }).collect();
	338	+ out.sort_by(\|a, b\| b.bytes.cmp(&a.bytes).then_with(\|\| a.name.cmp(&b.name)));
	339	+ Ok(out)
	340	+}
	341	+
	342	+/// The cached breakdown for the repo's default branch, recomputed when the
	343	+/// branch has moved since. Also keeps `repos.primary_language` current.
	344	+pub async fn for_repo(state: &AppState, repo: &Repo) -> anyhow::Result<Vec<LanguageBytes>> {
	345	+ let git = Git::new(repo.disk_path(&state.config));
	346	+ let Some(head) = git.resolve_commit(&format!("refs/heads/{}", repo.default_branch)).await? else {
	347	+ return Ok(Vec::new());
	348	+ };
	349	+ let cached: Option<(String, sqlx::types::Json<Vec<LanguageBytes>>)> =
	350	+ sqlx::query_as("select commit_sha, stats from repo_languages where repo_id = $1").bind(repo.id).fetch_optional(&state.db).await?;
	351	+ if let Some((sha, stats)) = cached {
	352	+ if sha == head {
	353	+ return Ok(stats.0);
	354	+ }
	355	+ }
	356	+ let started = std::time::Instant::now();
	357	+ let stats = compute(&git, &head).await?;
	358	+ let primary = stats.first().map(\|l\| l.name.clone());
	359	+ sqlx::query(
	360	+ "insert into repo_languages (repo_id, commit_sha, stats) values ($1, $2, $3)
	361	+ on conflict (repo_id) do update set commit_sha = excluded.commit_sha, stats = excluded.stats, computed_at = now()",
	362	+ )
	363	+ .bind(repo.id)
	364	+ .bind(&head)
	365	+ .bind(sqlx::types::Json(&stats))
	366	+ .execute(&state.db)
	367	+ .await?;
	368	+ sqlx::query("update repos set primary_language = $2 where id = $1").bind(repo.id).bind(&primary).execute(&state.db).await?;
	369	+ tracing::debug!(repo = %repo.full_name(), languages = stats.len(), ms = started.elapsed().as_millis() as u64, "language stats computed");
	370	+ Ok(stats)
	371	+}
	372	+
	373	+/// Computes stats for repos that have none yet (after an upgrade), quietly in
	374	+/// the background, one at a time.
	375	+pub fn spawn_backfill(state: AppState) {
	376	+ tokio::spawn(async move {
	377	+ let repos: Vec<Repo> = match sqlx::query_as(concat!(
	378	+ crate::repo_select!(),
	379	+ " where not r.is_empty and not exists (select 1 from repo_languages l where l.repo_id = r.id) order by r.id"
	380	+ ))
	381	+ .fetch_all(&state.db)
	382	+ .await
	383	+ {
	384	+ Ok(repos) => repos,
	385	+ Err(error) => {
	386	+ tracing::warn!(%error, "language backfill query failed");
	387	+ return;
	388	+ }
	389	+ };
	390	+ if repos.is_empty() {
	391	+ return;
	392	+ }
	393	+ tracing::info!(repos = repos.len(), "computing language stats for existing repositories");
	394	+ for repo in repos {
	395	+ if let Err(error) = for_repo(&state, &repo).await {
	396	+ tracing::warn!(repo = %repo.full_name(), ?error, "language stats failed");
	397	+ }
	398	+ }
	399	+ });
	400	+}
	401	+
	402	+/// Percentages for display: languages under 0.1% are folded into "Other".
	403	+pub fn percentages(stats: &[LanguageBytes]) -> Vec<(String, f64)> {
	404	+ let total: u64 = stats.iter().map(\|l\| l.bytes).sum();
	405	+ if total == 0 {
	406	+ return Vec::new();
	407	+ }
	408	+ let mut out = Vec::new();
	409	+ let mut other = 0.0;
	410	+ for language in stats {
	411	+ let pct = language.bytes as f64 * 100.0 / total as f64;
	412	+ if pct < 0.1 \|\| out.len() >= 8 {
	413	+ other += pct;
	414	+ } else {
	415	+ out.push((language.name.clone(), pct));
	416	+ }
	417	+ }
	418	+ if other >= 0.05 {
	419	+ out.push(("Other".to_string(), other));
	420	+ }
	421	+ out
	422	+}
	423	+
	424	+#[cfg(test)]
	425	+mod tests {
	426	+ use super::*;
	427	+
	428	+ #[test]
	429	+ fn detects_by_name_and_extension() {
	430	+ assert_eq!(detect("src/main.rs"), Some("Rust"));
	431	+ assert_eq!(detect("web/App.tsx"), Some("TypeScript"));
	432	+ assert_eq!(detect("Dockerfile"), Some("Dockerfile"));
	433	+ assert_eq!(detect("deploy/Dockerfile.prod"), Some("Dockerfile"));
	434	+ assert_eq!(detect("frontend/src/pages/index.astro"), Some("Astro"));
	435	+ assert_eq!(detect("README.md"), None);
	436	+ assert_eq!(detect("package.json"), None);
	437	+ assert_eq!(detect("config.yaml"), None);
	438	+ }
	439	+
	440	+ #[test]
	441	+ fn vendored_and_docs_are_excluded() {
	442	+ assert!(excluded_by_default("node_modules/react/index.js"));
	443	+ assert!(excluded_by_default("app/vendor/lib.rb"));
	444	+ assert!(excluded_by_default("static/jquery.min.js"));
	445	+ assert!(excluded_by_default("docs/conf.py"));
	446	+ assert!(!excluded_by_default("src/docs/page.rs"));
	447	+ assert!(!excluded_by_default("src/lib.rs"));
	448	+ }
	449	+
	450	+ #[test]
	451	+ fn gitattributes_override() {
	452	+ let overrides = Overrides::parse(".js linguist-vendored\ngen/* linguist-generated\n*.inc linguist-language=PHP\nkeep.js -linguist-vendored\n");
	453	+ assert_eq!(overrides.apply("a/b.js", false, Some("JavaScript")), (true, Some("JavaScript")));
	454	+ assert_eq!(overrides.apply("keep.js", false, Some("JavaScript")), (false, Some("JavaScript")));
	455	+ assert!(overrides.apply("gen/x.rs", false, Some("Rust")).0);
	456	+ assert_eq!(overrides.apply("lib/a.inc", false, None), (false, Some("PHP")));
	457	+ }
	458	+
	459	+ #[test]
	460	+ fn small_languages_fold_into_other() {
	461	+ let stats = vec![
	462	+ LanguageBytes { name: "Rust".into(), bytes: 9000 },
	463	+ LanguageBytes { name: "Shell".into(), bytes: 995 },
	464	+ LanguageBytes { name: "Nix".into(), bytes: 5 },
	465	+ ];
	466	+ let pct = percentages(&stats);
	467	+ assert_eq!(pct[0].0, "Rust");
	468	+ assert!((pct[0].1 - 90.0).abs() < 0.01);
	469	+ assert_eq!(pct.last().unwrap().0, "Other");
	470	+ }
	471	+}

+1-0backend/src/main.rs

		@@ -20,6 +20,7 @@ mod git;
20	20	mod git_http;
21	21	mod hook;
22	22	mod importer;
	23	+mod languages;
23	24	mod jobs;
24	25	mod lfs;
25	26	mod logging;

+3-1backend/src/models.rs

		@@ -77,6 +77,8 @@ pub struct Repo {
77	77	pub created_at: DateTime<Utc>,
78	78	pub updated_at: DateTime<Utc>,
79	79	pub pushed_at: Option<DateTime<Utc>>,
	80	+ /// Largest language on the default branch (see `languages`).
	81	+ pub primary_language: Option<String>,
80	82	}
81	83
82	84	/// `select ... from repos r join accounts a` yielding `Repo` rows; append
		@@ -85,7 +87,7 @@ pub struct Repo {
85	87	macro_rules! repo_select {
86	88	() => {
87	89	"select r.id, r.owner_id, a.name as owner_name, a.kind as owner_kind, r.name, r.description, r.visibility, \
88		- r.default_branch, r.size_bytes, r.is_empty, r.archived, r.created_at, r.updated_at, r.pushed_at \
	90	+ r.default_branch, r.size_bytes, r.is_empty, r.archived, r.created_at, r.updated_at, r.pushed_at, r.primary_language \
89	91	from repos r join accounts a on a.id = r.owner_id"
90	92	};
91	93	}

+5-0backend/src/push.rs

		@@ -115,6 +115,11 @@ pub async fn after_push(state: &AppState, repo: &Repo, pusher: Option<&Viewer>,
115	115
116	116	if update.name == default_ref && update.new != ZERO_SHA {
117	117	record_contributions(state, repo.id, &git, &update.old, &update.new).await?;
	118	+ let mut moved = repo.clone();
	119	+ moved.default_branch = default_branch.clone();
	120	+ if let Err(error) = crate::languages::for_repo(state, &moved).await {
	121	+ tracing::warn!(repo = %repo.full_name(), ?error, "language stats failed");
	122	+ }
118	123	}
119	124	}
120	125

+1-0backend/src/web/components.rs

		@@ -26,6 +26,7 @@ pub fn repo_row(repo: &Repo, show_owner: bool) -> Markup {
26	26	p class="mt-0.5 line-clamp-2 text-[13px] text-ink-dim" { (repo.description) }
27	27	}
28	28	div class="mt-1 flex flex-wrap gap-x-3 text-xs text-ink-faint" {
	29	+ @if let Some(language) = &repo.primary_language { span class="text-ink-dim" { (ui::language(language)) } }
29	30	@if let Some(pushed) = repo.pushed_at { span { "Updated " (ui::time(pushed)) } } @else { span { "Empty" } }
30	31	@if repo.size_bytes > 0 { span { (ui::bytes(repo.size_bytes as u64)) } }
31	32	}

+35-1backend/src/web/profile.rs

		@@ -103,6 +103,29 @@ async fn counts(ctx: &Ctx, owner_id: i64) -> AppResult<(i64, i64)> {
103	103	Ok((repos, packages))
104	104	}
105	105
	106	+/// Bytes per language across an account's repositories the viewer can see.
	107	+pub async fn language_totals(ctx: &Ctx, owner_id: i64) -> AppResult<Vec<crate::languages::LanguageBytes>> {
	108	+ let (viewer_id, admin) = perm::visibility_binds(ctx.viewer.as_ref(), Area::Repo);
	109	+ let rows: Vec<(sqlx::types::Json<Vec<crate::languages::LanguageBytes>>,)> = sqlx::query_as(concat!(
	110	+ "select l.stats from repo_languages l join repos r on r.id = l.repo_id where r.owner_id = $3 and ",
	111	+ visible_repos!()
	112	+ ))
	113	+ .bind(viewer_id)
	114	+ .bind(admin)
	115	+ .bind(owner_id)
	116	+ .fetch_all(&ctx.state.db)
	117	+ .await?;
	118	+ let mut totals: HashMap<String, u64> = HashMap::new();
	119	+ for (stats,) in rows {
	120	+ for language in stats.0 {
	121	+ *totals.entry(language.name).or_default() += language.bytes;
	122	+ }
	123	+ }
	124	+ let mut out: Vec<crate::languages::LanguageBytes> = totals.into_iter().map(\|(name, bytes)\| crate::languages::LanguageBytes { name, bytes }).collect();
	125	+ out.sort_by(\|a, b\| b.bytes.cmp(&a.bytes).then_with(\|\| a.name.cmp(&b.name)));
	126	+ Ok(out)
	127	+}
	128	+
106	129	/// Days with contributions the viewer may see, for the rolling year or one
107	130	/// calendar year, plus the years that have any (for the year switcher).
108	131	pub async fn contributions(ctx: &Ctx, user_id: i64, year: Option<i32>) -> AppResult<(Contributions, Vec<i32>)> {
		@@ -305,6 +328,7 @@ async fn user_page(ctx: &Ctx, account: &Account, query: &ProfileQuery) -> AppRes
305	328	let selected_year = query.year.unwrap_or_else(\|\| data.today.year());
306	329	let popular = owned_repos(ctx, account.id, None, 6).await?;
307	330	let feed = activity(ctx, Some(account.id), None, 25).await?;
	331	+ let languages = language_totals(ctx, account.id).await?;
308	332	html! {
309	333	@if !popular.is_empty() {
310	334	h2 class="mb-2 text-[13px] font-semibold" { "Repositories" }
		@@ -317,7 +341,10 @@ async fn user_page(ctx: &Ctx, account: &Account, query: &ProfileQuery) -> AppRes
317	341	(ui::visibility_tag(&repo.visibility))
318	342	}
319	343	p class="mt-1 line-clamp-2 flex-1 text-xs text-ink-dim" { (repo.description) }
320		- @if let Some(pushed) = repo.pushed_at { div class="mt-1.5 text-[11px] text-ink-faint" { "Updated " (ui::time(pushed)) } }
	344	+ div class="mt-1.5 flex flex-wrap gap-x-3 text-[11px] text-ink-faint" {
	345	+ @if let Some(language) = &repo.primary_language { span class="text-ink-dim" { (ui::language(language)) } }
	346	+ @if let Some(pushed) = repo.pushed_at { span { "Updated " (ui::time(pushed)) } }
	347	+ }
321	348	}
322	349	}
323	350	}
		@@ -349,6 +376,13 @@ async fn user_page(ctx: &Ctx, account: &Account, query: &ProfileQuery) -> AppRes
349	376	}
350	377	}
351	378	}
	379	+ @if !languages.is_empty() {
	380	+ h2 class="mb-2 text-[13px] font-semibold" { "Languages" }
	381	+ div class="box mb-5 grid items-center gap-4 p-3 sm:grid-cols-[minmax(0,400px)_1fr]" {
	382	+ div class="flex justify-center" { (ui::language_radar(&languages)) }
	383	+ div class="min-w-0" { (ui::language_breakdown(&languages)) }
	384	+ }
	385	+ }
352	386	h2 class="mb-2 text-[13px] font-semibold" { "Activity" }
353	387	div class="box" { (activity_list(&feed, false)) }
354	388	}

+7-0backend/src/web/repo.rs

		@@ -332,6 +332,7 @@ async fn tree_view(ctx: &Ctx, loaded: &Loaded, git_ref: &str, commit: &str, path
332	332	visible_packages.push(package);
333	333	}
334	334	}
	335	+ let languages = if is_root { crate::languages::for_repo(&ctx.state, repo).await.unwrap_or_default() } else { Vec::new() };
335	336	let branch_count = refs.iter().filter(\|r\| r.is_branch()).count();
336	337	let tag_count = refs.iter().filter(\|r\| r.is_tag()).count();
337	338
		@@ -389,6 +390,12 @@ async fn tree_view(ctx: &Ctx, loaded: &Loaded, git_ref: &str, commit: &str, path
389	390	li { (ui::bytes(repo.size_bytes as u64)) " on disk" }
390	391	@if let Some(pushed) = repo.pushed_at { li { "Last push " (ui::time(pushed)) } }
391	392	}
	393	+ @if !languages.is_empty() {
	394	+ div {
	395	+ h2 class="mb-2 font-semibold" { "Languages" }
	396	+ (ui::language_breakdown(&languages))
	397	+ }
	398	+ }
392	399	@if !visible_packages.is_empty() {
393	400	div {
394	401	h2 class="mb-1 font-semibold" { "Images" }

+94-0backend/src/web/ui.rs

		@@ -93,6 +93,100 @@ pub fn plural(n: i64, one: &str, many: &str) -> String {
93	93	format!("{n} {}", if n == 1 { one } else { many })
94	94	}
95	95
	96	+/// A language's color swatch and name, as in repo lists.
	97	+pub fn language(name: &str) -> Markup {
	98	+ html! {
	99	+ span class="inline-flex items-center gap-1" {
	100	+ span class="inline-block h-2.5 w-2.5 shrink-0 rounded-[2px]" style={ "background:" (crate::languages::color(name)) } {}
	101	+ (name)
	102	+ }
	103	+ }
	104	+}
	105	+
	106	+/// GitHub-style language bar and legend for a repository.
	107	+pub fn language_breakdown(stats: &[crate::languages::LanguageBytes]) -> Markup {
	108	+ let rows = crate::languages::percentages(stats);
	109	+ html! {
	110	+ @if !rows.is_empty() {
	111	+ div class="flex h-2 overflow-hidden rounded-[3px] bg-surface-hover" role="img"
	112	+ aria-label={ @for (i, (name, pct)) in rows.iter().enumerate() { @if i > 0 { ", " } (name) " " (format!("{pct:.1}")) "%" } } {
	113	+ @for (name, pct) in &rows {
	114	+ span class="h-full" style={ "width:" (format!("{pct:.3}")) "%;background:" (crate::languages::color(name)) } data-tip={ (name) " " (format!("{pct:.1}")) "%" } {}
	115	+ }
	116	+ }
	117	+ ul class="mt-2 flex flex-wrap gap-x-3 gap-y-1 text-xs" {
	118	+ @for (name, pct) in &rows {
	119	+ li class="inline-flex items-center gap-1" {
	120	+ span class="inline-block h-2.5 w-2.5 rounded-[2px]" style={ "background:" (crate::languages::color(name)) } {}
	121	+ span class="font-semibold text-ink" { (name) }
	122	+ span class="text-ink-dim" { (format!("{pct:.1}")) "%" }
	123	+ }
	124	+ }
	125	+ }
	126	+ }
	127	+ }
	128	+}
	129	+
	130	+/// Spider chart of a language mix: one axis per language (largest first,
	131	+/// clockwise from the top). Shares sit on a log scale (0.05% at the centre,
	132	+/// the largest share at the rim): language mixes are lopsided, and on a
	133	+/// linear scale everything but the main language collapses into the middle.
	134	+/// Labels and tooltips carry the exact shares, and a caption names the
	135	+/// scale. One series in the accent hue; needs at least three languages.
	136	+pub fn language_radar(stats: &[crate::languages::LanguageBytes]) -> Markup {
	137	+ let total: u64 = stats.iter().map(\|l\| l.bytes).sum();
	138	+ let axes: Vec<(&str, f64, u64)> = stats.iter().take(8).map(\|l\| (l.name.as_str(), l.bytes as f64 * 100.0 / total.max(1) as f64, l.bytes)).collect();
	139	+ if axes.len() < 3 {
	140	+ return html! {};
	141	+ }
	142	+ let max = axes.iter().map(\|a\| a.1).fold(0.0_f64, f64::max).max(0.1);
	143	+ const FLOOR: f64 = 0.05;
	144	+ let scale = \|share: f64\| ((share.max(FLOOR) / FLOOR).log10() / (max / FLOOR).log10()).clamp(0.04, 1.0);
	145	+ let pct = \|share: f64\| if share < 10.0 { format!("{share:.1}%") } else { format!("{share:.0}%") };
	146	+ let (cx, cy, r) = (160.0_f64, 135.0_f64, 92.0_f64);
	147	+ let n = axes.len() as f64;
	148	+ let point = \|i: usize, fraction: f64\| {
	149	+ let angle = -std::f64::consts::FRAC_PI_2 + i as f64 * std::f64::consts::TAU / n;
	150	+ (cx + angle.cos() * r * fraction, cy + angle.sin() * r * fraction)
	151	+ };
	152	+ let ring = \|fraction: f64\| -> String {
	153	+ (0..axes.len()).map(\|i\| { let (x, y) = point(i, fraction); format!("{x:.1},{y:.1}") }).collect::<Vec<_>>().join(" ")
	154	+ };
	155	+ let shape: String = axes.iter().enumerate().map(\|(i, a)\| { let (x, y) = point(i, scale(a.1)); format!("{x:.1},{y:.1}") }).collect::<Vec<_>>().join(" ");
	156	+ html! {
	157	+ figure class="m-0 w-full max-w-[400px]" {
	158	+ // Wide side margins: axis labels such as "Jupyter Notebook" sit outside the rim.
	159	+ svg viewBox="-50 0 420 270" class="block h-auto w-full" role="img"
	160	+ aria-label={ "Languages: " @for (i, a) in axes.iter().enumerate() { @if i > 0 { ", " } (a.0) " " (format!("{:.1}", a.1)) "%" } } {
	161	+ // Recessive grid: hairline rings at quarter steps of the largest share, and spokes.
	162	+ @for fraction in [0.25, 0.5, 0.75, 1.0] {
	163	+ polygon points=(ring(fraction)) fill="none" class="stroke-edge" stroke-width="1" {}
	164	+ }
	165	+ @for i in 0..axes.len() {
	166	+ @let (x, y) = point(i, 1.0);
	167	+ line x1=(cx) y1=(cy) x2=(format!("{x:.1}")) y2=(format!("{y:.1}")) class="stroke-edge" stroke-width="1" {}
	168	+ }
	169	+ polygon points=(shape) class="fill-ember/15 stroke-ember" stroke-width="2" stroke-linejoin="round" {}
	170	+ @for (i, a) in axes.iter().enumerate() {
	171	+ @let (x, y) = point(i, scale(a.1));
	172	+ @let tip = format!("{} {:.1}% ({})", a.0, a.1, bytes(a.2));
	173	+ circle cx=(format!("{x:.1}")) cy=(format!("{y:.1}")) r="4" class="fill-ember stroke-surface" stroke-width="2" {}
	174	+ // Larger invisible hit target carrying the tooltip.
	175	+ circle cx=(format!("{x:.1}")) cy=(format!("{y:.1}")) r="12" fill="transparent" data-tip=(tip) {}
	176	+ @let (lx, ly) = point(i, 1.0 + 16.0 / r);
	177	+ @let anchor = if (lx - cx).abs() < 6.0 { "middle" } else if lx > cx { "start" } else { "end" };
	178	+ @let baseline = if ly < cy - r * 0.6 { -2.0 } else if ly > cy + r * 0.6 { 10.0 } else { 4.0 };
	179	+ text x=(format!("{lx:.1}")) y=(format!("{:.1}", ly + baseline)) text-anchor=(anchor) font-size="11" class="fill-ink-dim" data-tip=(tip) {
	180	+ (a.0) " "
	181	+ tspan class="fill-ink-faint" { (pct(a.1)) }
	182	+ }
	183	+ }
	184	+ }
	185	+ figcaption class="mt-1 text-center text-[11px] text-ink-faint" { "Share of code, log scale" }
	186	+ }
	187	+ }
	188	+}
	189	+
96	190	pub fn visibility_tag(visibility: &str) -> Markup {
97	191	html! {
98	192	@if visibility == "private" {

+3-1cli/src/cmds.rs

		@@ -92,13 +92,14 @@ pub fn repo_list(s: &Session, owner: Option<&str>) -> anyhow::Result<()> {
92	92	r.full_name.clone(),
93	93	format!("{}{}", r.visibility, if r.archived { ", archived" } else { "" }),
94	94	if r.is_empty { "-".into() } else { r.default_branch.clone() },
	95	+ r.language.clone().unwrap_or_else(\|\| "-".into()),
95	96	bytes(r.size_bytes),
96	97	ago(r.pushed_at),
97	98	r.description.chars().take(60).collect(),
98	99	]
99	100	})
100	101	.collect();
101		- table(&["NAME", "VISIBILITY", "BRANCH", "SIZE", "PUSHED", "DESCRIPTION"], &rows);
	102	+ table(&["NAME", "VISIBILITY", "BRANCH", "LANGUAGE", "SIZE", "PUSHED", "DESCRIPTION"], &rows);
102	103	Ok(())
103	104	}
104	105
		@@ -118,6 +119,7 @@ pub fn repo_view(s: &Session, spec: &str) -> anyhow::Result<()> {
118	119	vec!["https".into(), repo.clone_https.clone()],
119	120	vec!["ssh".into(), repo.clone_ssh.clone()],
120	121	vec!["default branch".into(), if repo.is_empty { "(empty repository)".into() } else { repo.default_branch.clone() }],
	122	+ vec!["language".into(), repo.language.clone().unwrap_or_else(\|\| "-".into())],
121	123	vec!["size".into(), bytes(repo.size_bytes)],
122	124	vec!["last push".into(), ago(repo.pushed_at)],
123	125	vec!["your access".into(), repo.permission.clone().unwrap_or_else(\|\| "none".into())],

+3-0shared/src/lib.rs

		@@ -57,6 +57,9 @@ pub struct Repo {
57	57	/// "public" or "private".
58	58	pub visibility: String,
59	59	pub default_branch: String,
	60	+ /// Largest language on the default branch, when known.
	61	+ #[serde(default)]
	62	+ pub language: Option<String>,
60	63	pub size_bytes: i64,
61	64	pub is_empty: bool,
62	65	pub archived: bool,