irongit
Sign inRegister
huncholanehuncholane/irongitPublic

Git hosting and a container registry in one Rust binary (axum + Astro)

irongit/scripts/gen-linguist.py
121 lines4.9 KBPython
RawDownloadHistory
1#!/usr/bin/env python3
2"""Build backend/src/languages/linguist.json from GitHub Linguist's data files.
3
4irongit detects languages the way GitHub does, with Linguist's own tables:
5languages.yml (names, colors, groups, extensions, filenames, interpreters,
6aliases), vendor.yml and documentation.yml (excluded paths), generic.yml
7(extensions too vague to decide on), and heuristics.yml (content rules for
8shared extensions such as .h, .m, .sql or .pl). This script downloads them for
9one Linguist release and writes a compact JSON the server embeds.
10
11 python3 scripts/gen-linguist.py # the release pinned below
12 python3 scripts/gen-linguist.py v9.8.0 # another release
13
14Heuristic patterns are Ruby regexes; they are rewritten for Rust's regex crate
15here (Ruby's ^ and $ always match at line breaks, Ruby's m flag is Rust's s,
16a stray { is a literal). A few use lookarounds or subroutines Rust lacks; a
17server test checks that those only choose between data and prose formats,
18which never count, so a pattern that never matches costs nothing.
19
20Linguist is MIT licensed: https://github.com/github-linguist/linguist
21"""
22
23import json
24import re
25import sys
26import urllib.request
27from pathlib import Path
28
29import yaml
30
31TAG = sys.argv[1] if len(sys.argv) > 1 else "v9.7.0"
32BASE = f"https://raw.githubusercontent.com/github-linguist/linguist/{TAG}/lib/linguist"
33OUT = Path(__file__).resolve().parent.parent / "backend/src/languages/linguist.json"
34
35
36def fetch(name):
37 with urllib.request.urlopen(f"{BASE}/{name}", timeout=30) as response:
38 return yaml.safe_load(response.read())
39
40
41def ruby_to_rust(pattern):
42 """One Ruby regex as a Rust one with the same meaning."""
43 # Ruby's inline m flag is dot-matches-newline, which Rust calls s.
44 def flags(match):
45 on, sep = match.group(1), match.group(2)
46 return "(?" + on.replace("m", "s") + sep
47 pattern = re.sub(r"\(\?([imx-]+)([:)])", flags, pattern)
48 # {,n} is Ruby shorthand for {0,n}.
49 pattern = re.sub(r"\{,(\d+)\}", r"{0,\1}", pattern)
50 # Ruby reads a { that starts no repetition as a literal; Rust wants \{.
51 pattern = re.sub(r"(?<!\\)\{(?!\d+(?:,\d*)?\})", r"\\{", pattern)
52 # Rust has no lookbehind; "not after {" is spelled out.
53 pattern = pattern.replace("(?<!\\{)", "(?:^|[^{])")
54 # Ruby's ^ and $ are always line anchors.
55 return "(?m)" + pattern
56
57
58def union(value):
59 patterns = value if isinstance(value, list) else [value]
60 if len(patterns) == 1:
61 return ruby_to_rust(patterns[0])
62 return "|".join(f"(?:{ruby_to_rust(p)})" for p in patterns)
63
64
65def main():
66 languages = fetch("languages.yml")
67 vendor = fetch("vendor.yml")
68 documentation = fetch("documentation.yml")
69 generic = fetch("generic.yml")
70 heuristics = fetch("heuristics.yml")
71
72 kinds = {"programming": "programming", "markup": "markup", "data": "data", "prose": "prose"}
73 out_languages = []
74 for name, spec in languages.items():
75 entry = {"name": name, "type": kinds[spec["type"]]}
76 if spec.get("color"):
77 entry["color"] = spec["color"]
78 if spec.get("group"):
79 entry["group"] = spec["group"]
80 for key, short in (("extensions", "ext"), ("filenames", "files"), ("interpreters", "interp"), ("aliases", "aliases")):
81 if spec.get(key):
82 entry[short] = spec[key]
83 out_languages.append(entry)
84
85 named = {key: union(value) for key, value in heuristics["named_patterns"].items()}
86
87 def conjuncts(rule):
88 if rule.get("and") is not None:
89 return [c for block in rule["and"] for c in conjuncts(block)]
90 if rule.get("pattern") is not None:
91 return [{"re": union(rule["pattern"])}]
92 if rule.get("negative_pattern") is not None:
93 return [{"re": union(rule["negative_pattern"]), "neg": True}]
94 if rule.get("named_pattern") is not None:
95 return [{"re": named[rule["named_pattern"]]}]
96 return []
97
98 out_heuristics = []
99 for block in heuristics["disambiguations"]:
100 rules = []
101 for rule in block["rules"]:
102 names = rule["language"] if isinstance(rule["language"], list) else [rule["language"]]
103 rules.append({"languages": names, "all": conjuncts(rule)})
104 out_heuristics.append({"ext": block["extensions"], "rules": rules})
105
106 data = {
107 "source": f"github-linguist/linguist {TAG}",
108 "languages": out_languages,
109 "vendor": vendor,
110 "documentation": documentation,
111 "generic": generic["extensions"],
112 "heuristics": out_heuristics,
113 }
114 OUT.parent.mkdir(parents=True, exist_ok=True)
115 OUT.write_text(json.dumps(data, separators=(",", ":"), ensure_ascii=False) + "\n")
116 print(f"wrote {OUT} ({OUT.stat().st_size // 1024} KiB): {len(out_languages)} languages, "
117 f"{len(vendor)} vendor and {len(documentation)} documentation paths, {len(out_heuristics)} heuristics")
118
119
120if __name__ == "__main__":
121 main()