the static git browser that builds this site
git clone https://git.lucas.co/gitsite.git
generate.py (11.3K)
1 #!/usr/bin/env python3
2 # Static git repository browser generator for git.lucas.co.
3 # Reads bare mirrors from ~/.cache/gitsite/mirrors (created by build.sh)
4 # and writes a browsable HTML site to ~/.cache/gitsite/out.
5
6 import html
7 import os
8 import shutil
9 import subprocess
10 import sys
11 from pathlib import Path
12 from urllib.parse import quote
13
14 BASE = Path(__file__).resolve().parent
15 CACHE = Path.home() / ".cache" / "gitsite"
16 MIRRORS = CACHE / "mirrors"
17 OUT = CACHE / "out"
18
19 SITE_TITLE = "git.lucas.co"
20 HOME_URL = "https://lucas.co"
21 CLONE_BASE = "https://git.lucas.co"
22
23 MAX_DIFF_BYTES = 500_000 # truncate commit patches beyond this
24 MAX_BLOB_BYTES = 300_000 # don't render file contents beyond this
25
26
27 def read_repos():
28 repos = []
29 for line in (BASE / "repos.conf").read_text().splitlines():
30 line = line.strip()
31 if not line or line.startswith("#"):
32 continue
33 name, path, desc, mode = line.split("|")
34 repos.append({"name": name, "path": path, "desc": desc,
35 "clone": mode == "clone"})
36 return repos
37
38
39 def git(mirror, *args, binary=False):
40 r = subprocess.run(["git", "-C", str(mirror), *args], capture_output=True)
41 if r.returncode != 0:
42 raise RuntimeError(f"git {' '.join(args)} failed in {mirror}: "
43 f"{r.stderr.decode('utf-8', 'replace')}")
44 return r.stdout if binary else r.stdout.decode("utf-8", "replace")
45
46
47 def esc(s):
48 return html.escape(s, quote=True)
49
50
51 def human_size(n):
52 if n == "-": # e.g. submodule entries
53 return "-"
54 n = int(n)
55 if n < 1024:
56 return f"{n}B"
57 for unit in ("K", "M", "G"):
58 n /= 1024
59 if n < 1024 or unit == "G":
60 return f"{n:.1f}".removesuffix(".0") + unit
61
62
63 def write_page(path, title, body, site_root):
64 path.parent.mkdir(parents=True, exist_ok=True)
65 path.write_text(f"""<!DOCTYPE html>
66 <html>
67 <head>
68 <meta charset="UTF-8">
69 <meta name="viewport" content="width=device-width, initial-scale=1.0">
70 <title>{esc(title)}</title>
71 <link rel="stylesheet" href="{site_root}style.css">
72 </head>
73 <body>
74 {body}
75 </body>
76 </html>
77 """)
78
79
80 def rel(from_dir, to_dir):
81 # relative prefix ("", "../", "../../", ...) from a page dir to another dir
82 r = os.path.relpath(to_dir, from_dir)
83 return "" if r == "." else r + "/"
84
85
86 def repo_header(repo, page_dir, repo_dir, active):
87 site_root = rel(page_dir, OUT)
88 repo_root = rel(page_dir, repo_dir)
89 nav = " | ".join(
90 f'<a href="{repo_root}{href}">{label}</a>' if label != active
91 else f'<span class="active">{label}</span>'
92 for label, href in (("Log", "index.html"), ("Files", "files.html"),
93 ("Refs", "refs.html")))
94 desc = f'\n<div class="desc">{esc(repo["desc"])}</div>' if repo["desc"] else ""
95 clone = (f'\n<div class="clone">git clone {CLONE_BASE}/{repo["name"]}.git</div>'
96 if repo["clone"] else "")
97 return (f'<div class="crumbs"><a href="{site_root}index.html">{SITE_TITLE}</a>'
98 f' / <a href="{repo_root}index.html">{esc(repo["name"])}</a></div>{desc}{clone}\n'
99 f'<div class="nav">{nav}</div>\n<hr>\n')
100
101
102 def fmt_diff(patch):
103 out = []
104 for line in patch.split("\n"):
105 e = esc(line)
106 if line.startswith("diff --git"):
107 out.append(f'<span class="df">{e}</span>')
108 elif line.startswith("@@"):
109 out.append(f'<span class="dh">{e}</span>')
110 elif line.startswith("+") and not line.startswith("+++"):
111 out.append(f'<span class="di">{e}</span>')
112 elif line.startswith("-") and not line.startswith("---"):
113 out.append(f'<span class="dd">{e}</span>')
114 else:
115 out.append(e)
116 return "\n".join(out)
117
118
119 def gen_commit_pages(repo, mirror, repo_dir, commits):
120 page_dir = repo_dir / "commit"
121 for h, at, author, subject in commits:
122 meta = git(mirror, "show", "--no-patch",
123 "--format=%H%x1f%P%x1f%an <%ae>%x1f%ad%x1f%B",
124 "--date=format:%Y-%m-%d %H:%M", h)
125 full, parents, who, date, msg = meta.split("\x1f", 4)
126 patch_b = git(mirror, "show", "--format=", "--stat", "--patch",
127 "--no-color", h, binary=True)
128 truncated = len(patch_b) > MAX_DIFF_BYTES
129 patch = patch_b[:MAX_DIFF_BYTES].decode("utf-8", "replace")
130 parent_html = " ".join(
131 f'<a href="{p}.html">{p[:10]}</a>' for p in parents.split() if p)
132 body = repo_header(repo, page_dir, repo_dir, None)
133 body += '<table class="meta">\n'
134 body += f'<tr><td>commit</td><td>{full}</td></tr>\n'
135 if parent_html:
136 body += f'<tr><td>parent</td><td>{parent_html}</td></tr>\n'
137 body += f'<tr><td>author</td><td>{esc(who)}</td></tr>\n'
138 body += f'<tr><td>date</td><td>{date}</td></tr>\n'
139 body += '</table>\n'
140 body += f'<pre class="msg">{esc(msg.strip())}</pre>\n<hr>\n'
141 body += f'<pre class="diff">{fmt_diff(patch)}</pre>\n'
142 if truncated:
143 body += '<div class="notice">diff truncated</div>\n'
144 write_page(page_dir / f"{full}.html",
145 f'{repo["name"]}: {subject}', body, rel(page_dir, OUT))
146
147
148 def gen_log(repo, mirror, repo_dir, commits):
149 body = repo_header(repo, repo_dir, repo_dir, "Log")
150 body += '<table class="list">\n<tr><td>Date</td><td>Message</td><td>Author</td></tr>\n'
151 for h, at, author, subject in commits:
152 body += (f'<tr><td>{at}</td>'
153 f'<td><a href="commit/{h}.html">{esc(subject)}</a></td>'
154 f'<td>{esc(author)}</td></tr>\n')
155 body += '</table>\n'
156 if not commits:
157 body += '<div class="notice">no commits yet</div>\n'
158 write_page(repo_dir / "index.html", repo["name"], body, rel(repo_dir, OUT))
159
160
161 def gen_files(repo, mirror, repo_dir):
162 body = repo_header(repo, repo_dir, repo_dir, "Files")
163 body += '<table class="list">\n<tr><td>Mode</td><td>Name</td><td>Size</td></tr>\n'
164 entries = []
165 try:
166 tree = git(mirror, "ls-tree", "-r", "-l", "HEAD").splitlines()
167 except RuntimeError: # empty repository
168 tree = []
169 for line in tree:
170 info, path = line.split("\t", 1)
171 mode, otype, _h, size = info.split()
172 entries.append((mode, otype, size, path))
173 href = quote(f"file/{path}.html")
174 body += (f'<tr><td class="mode">{mode}</td>'
175 f'<td><a href="{href}">{esc(path)}</a></td>'
176 f'<td class="size">{human_size(size)}</td></tr>\n')
177 body += '</table>\n'
178 write_page(repo_dir / "files.html", f'{repo["name"]} files', body,
179 rel(repo_dir, OUT))
180 return entries
181
182
183 def gen_blob_pages(repo, mirror, repo_dir, entries):
184 for mode, otype, size, path in entries:
185 out_path = repo_dir / "file" / (path + ".html")
186 page_dir = out_path.parent
187 body = repo_header(repo, page_dir, repo_dir, None)
188 body += f'<div class="path">{esc(path)} ({human_size(size)})</div>\n<hr>\n'
189 if otype != "blob":
190 body += '<div class="notice">not a regular file</div>\n'
191 else:
192 content = git(mirror, "cat-file", "blob", f"HEAD:{path}", binary=True)
193 if b"\0" in content[:8000]:
194 body += '<div class="notice">binary file</div>\n'
195 elif len(content) > MAX_BLOB_BYTES:
196 body += '<div class="notice">file too large to display</div>\n'
197 else:
198 text = content.decode("utf-8", "replace")
199 lines = text.split("\n")
200 if lines and lines[-1] == "":
201 lines.pop()
202 w = len(str(len(lines)))
203 rows = "\n".join(
204 f'<a class="ln" id="l{i}" href="#l{i}">{i:>{w}}</a> {esc(l)}'
205 for i, l in enumerate(lines, 1))
206 body += f'<pre class="blob">{rows}</pre>\n'
207 write_page(out_path, f'{repo["name"]}: {path}', body, rel(page_dir, OUT))
208
209
210 def gen_refs(repo, mirror, repo_dir):
211 body = repo_header(repo, repo_dir, repo_dir, "Refs")
212 for title, pattern in (("Branches", "refs/heads"), ("Tags", "refs/tags")):
213 refs = git(mirror, "for-each-ref", "--sort=-creatordate",
214 "--format=%(refname:short)%1f%(objectname:short)%1f%(creatordate:short)",
215 pattern).splitlines()
216 if not refs and pattern == "refs/tags":
217 continue
218 body += f'<div class="section">{title}</div>\n<table class="list">\n'
219 for r in refs:
220 name, obj, date = r.split("\x1f")
221 body += f'<tr><td>{esc(name)}</td><td>{obj}</td><td>{date}</td></tr>\n'
222 body += '</table>\n'
223 write_page(repo_dir / "refs.html", f'{repo["name"]} refs', body,
224 rel(repo_dir, OUT))
225
226
227 def gen_404():
228 # a real 404.html switches Pages out of SPA-fallback mode; without it,
229 # unknown paths (e.g. git probing loose objects) get index.html with a 200
230 body = (f'<div class="crumbs"><a href="index.html">{SITE_TITLE}</a></div>\n'
231 '<hr>\n<div class="notice">not found</div>\n')
232 write_page(OUT / "404.html", f"{SITE_TITLE}: not found", body, "")
233
234
235 def gen_headers(repos):
236 # keep Cloudflare from recompressing/transforming git transport files
237 rules = ""
238 for repo in repos:
239 if repo["clone"]:
240 rules += (f'/{repo["name"]}.git/*\n'
241 " Cache-Control: no-transform\n"
242 " Content-Type: application/octet-stream\n")
243 (OUT / "_headers").write_text(rules)
244
245
246 def gen_index(repos):
247 body = (f'<div class="crumbs">{SITE_TITLE}</div>\n'
248 f'<div class="desc"><a href="{HOME_URL}">Lucas Galante</a>\'s projects</div>\n<hr>\n')
249 body += '<table class="list">\n<tr><td>Name</td><td>Description</td><td>Last commit</td></tr>\n'
250 for repo in repos:
251 mirror = MIRRORS / f'{repo["name"]}.git'
252 try:
253 last = git(mirror, "log", "-1", "--format=%as", "HEAD").strip()
254 except RuntimeError: # empty repository
255 last = "-"
256 body += (f'<tr><td><a href="{quote(repo["name"])}/index.html">{esc(repo["name"])}</a></td>'
257 f'<td>{esc(repo["desc"])}</td><td>{last}</td></tr>\n')
258 body += '</table>\n'
259 write_page(OUT / "index.html", SITE_TITLE, body, "")
260
261
262 def main():
263 repos = read_repos()
264 if OUT.exists():
265 shutil.rmtree(OUT)
266 OUT.mkdir(parents=True)
267 shutil.copy(BASE / "style.css", OUT / "style.css")
268 font = BASE / "font"
269 if font.is_dir():
270 shutil.copytree(font, OUT / "font")
271
272 for repo in repos:
273 mirror = MIRRORS / f'{repo["name"]}.git'
274 if not mirror.is_dir():
275 sys.exit(f"missing mirror {mirror}; run build.sh")
276 repo_dir = OUT / repo["name"]
277 commits = []
278 try:
279 log = git(mirror, "log", "--format=%H%x1f%as%x1f%an%x1f%s", "HEAD")
280 except RuntimeError: # empty repository
281 log = ""
282 for line in log.splitlines():
283 h, at, author, subject = line.split("\x1f")
284 commits.append((h, at, author, subject))
285 gen_log(repo, mirror, repo_dir, commits)
286 gen_commit_pages(repo, mirror, repo_dir, commits)
287 entries = gen_files(repo, mirror, repo_dir)
288 gen_blob_pages(repo, mirror, repo_dir, entries)
289 gen_refs(repo, mirror, repo_dir)
290 print(f'{repo["name"]}: {len(commits)} commits, {len(entries)} files')
291
292 gen_404()
293 gen_headers(repos)
294 gen_index(repos)
295 print(f"wrote {OUT}")
296
297
298 if __name__ == "__main__":
299 main()