publish / gen-browse.py
1,215 words
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
#!/usr/bin/env python3
"""Generates a static source browser for a published repo.

  gen-browse.py --repo <path> --app <name> --out <dir> \
                [--clone-url <url>] [--css <url>]

Reads HEAD of --repo (tracked content only — the same bytes the published
mirror serves) and writes <out>/index.html + one page per file:

  * code pages are syntax-highlighted with Pygments (pre-colored static
    HTML — no JS); plain escaped <pre> when Pygments is missing
  * markdown files are rendered to HTML (python-markdown); a directory's
    README.md is also rendered inline under its listing, GitHub-style
  * binary, oversized (>200 KB) and lockfile pages are stubs pointing at
    the git clone

Styling comes from one shared stylesheet (--css, default the ecosystem's
CDN) so every repo browser looks the same and pages stay tiny.
"""
import argparse
import base64
import html
import posixpath
import json
import re
import subprocess
from pathlib import Path

try:
    from pygments import highlight
    from pygments.formatters import HtmlFormatter
    from pygments.lexers import ClojureLexer, TextLexer, guess_lexer_for_filename
    from pygments.util import ClassNotFound

    FORMATTER = HtmlFormatter(linenos="table")
except ImportError:  # pragma: no cover
    highlight = None

try:
    import markdown

    def md_html(text: str) -> str:
        return markdown.markdown(
            text,
            extensions=["fenced_code", "tables", "codehilite"],
            extension_configs={"codehilite": {"css_class": "highlight",
                                              "guess_lang": False}},
        )
except ImportError:  # pragma: no cover
    md_html = None

LINK_RE = re.compile(r'(<a\s[^>]*href=")([^"#?]+)([^"]*")')


def relink_md(rendered: str, base_dir: str, tracked: set) -> str:
    """Point relative links at browse pages: a markdown link to a tracked
    file (README.md, src/foo.clj) 404s as-is — only <path>.html exists."""

    def fix(m):
        url = m.group(2)
        if "://" in url or url.startswith(("/", "mailto:", "#")):
            return m.group(0)
        target = posixpath.normpath(posixpath.join(base_dir, url))
        if target in tracked:
            return m.group(1) + url + ".html" + m.group(3)
        if (target + "/README.md" in tracked) or any(
            t.startswith(target + "/") for t in tracked
        ):  # directory link -> its listing
            return m.group(1) + url.rstrip("/") + "/index.html" + m.group(3)
        return m.group(0)

    return LINK_RE.sub(fix, rendered)


GENERATED = {"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "go.sum"}
MAX_RENDER = 200_000  # bytes; bigger files get a stub page

ap = argparse.ArgumentParser()
ap.add_argument("--repo", required=True)
ap.add_argument("--app", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--clone-url", default=None)
ap.add_argument("--css", default="https://cdn.cod-sursa.ro/browse.v1.css")
args = ap.parse_args()

APP = args.app
# ---- release coordinates -----------------------------------------------------
# A README's dependency snippet cannot know its own commit sha, so it says
# :git/sha "..." (or pins an npm semver range). The assembler writes the real
# coordinates of every published mirror to <site root>/releases.json; rendered
# markdown gets the same data-rel placeholders the landing pages use and a
# small script fills them client-side. The literal text stays the fallback.

def fill_releases(rendered: str, app: str) -> str:
    # highlighted code splits tokens into spans, so allow tags between them
    # sha placeholders: `:git/sha "..."` (deps.edn) and `revision: "..."` (SwiftPM)
    rendered = re.sub(
        r'((?::git/sha|\brevision)(?:\s|<[^>]+>|[:(])*?)(&quot;|")(?:\.\.\.|…)(&quot;|")',
        lambda m: f'{m.group(1)}{m.group(2)}<span data-rel="{app}.head">…</span>{m.group(3)}',
        rendered,
    )
    # npm git range: #semver:^X.Y.Z
    rendered = re.sub(
        r'(#semver:\^)(\d+\.\d+\.\d+)',
        lambda m: f'{m.group(1)}<span data-rel="{app}.version">{m.group(2)}</span>',
        rendered,
    )
    # SwiftPM ranges: from: "X.Y.Z" | exact: "X.Y.Z" | .upToNextMajor(from: "X.Y.Z")
    rendered = re.sub(
        r'(\b(?:from|exact)(?:\s|<[^>]+>)*?:(?:\s|<[^>]+>)*?)(&quot;|")(\d+\.\d+\.\d+)(&quot;|")',
        lambda m: f'{m.group(1)}{m.group(2)}<span data-rel="{app}.version">{m.group(3)}</span>{m.group(4)}',
        rendered,
    )
    return rendered


def releases_script(depth: int) -> str:
    """The fill script, or '' — pages are at <app>/browse/<path>, so the site
    root is two levels above the browse root."""
    url = "../" * depth + "../../releases.json"
    return (
        "<script>(function(){var els=document.querySelectorAll('[data-rel]');"
        "if(!els.length||!window.fetch)return;"
        f"fetch({json.dumps(url)})"
        ".then(function(r){return r.ok?r.json():null})"
        ".then(function(j){if(!j||!j.apps)return;"
        "Array.prototype.forEach.call(els,function(el){"
        "var k=el.getAttribute('data-rel').split('.'),a=j.apps[k[0]];var v=a&&a[k[1]];"
        "if(typeof v==='string'&&v)el.textContent=v;});})"
        ".catch(function(){});})();</script>"
    )

OUT = Path(args.out)
CLONE = args.clone_url or f"https://ird.cod-sursa.ro/{APP}.git"
HEAD_TAG = (
    f'<!doctype html><meta charset="utf-8">'
    f'<meta name="viewport" content="width=device-width,initial-scale=1">'
    f'<link rel="stylesheet" href="{args.css}">'
    '<style>.tools{display:flex;flex-wrap:wrap;gap:8px;align-items:center;margin:10px 0 14px}'
    '.tools button{font:inherit;font-size:13.5px;padding:6px 12px;border-radius:8px;cursor:pointer;'
    'border:1px solid currentColor;background:transparent;color:inherit;opacity:.85}'
    '.tools button:hover,.tools a.go:hover{opacity:1}'
    '.tools a.go{font-size:13.5px;padding:6px 12px;border-radius:8px;text-decoration:none;'
    'border:1px solid currentColor;font-weight:600}</style>'
)


def source_tools(text: str, name: str, depth: int) -> str:
    """Copy affordances for a rendered text file: a button that copies the
    raw source (embedded verbatim, so no fetch is needed on the gateway) and,
    for prompt templates (first line '# Prompt: …'), a link to the app's
    landing page where the FILL IN block is editable — the rendered page is
    for reading, the landing page is for copying."""
    stem = name.rsplit(".", 1)[0]
    up = "../" * depth
    go = ""
    if text.startswith("# Prompt:"):
        go = (f'<a class="go" href="{up}../index.html#{html.escape(stem)}">'
              f'edit FILL IN &amp; copy this prompt →</a>')
    # base64 keeps any byte sequence (including "</script>") inert inside
    # the page; decoded client-side on copy
    raw = base64.b64encode(text.encode("utf-8")).decode("ascii")
    return (
        f'<div class="tools">{go}<button type="button" data-copy="src">copy source</button>'
        f'<span class="muted" style="font-size:13px">{len(text.split()):,} words</span></div>'
        f'<script type="text/plain" id="src">{raw}</script>'
    )


COPY_SCRIPT = (
    "<script>(function(){var b=document.querySelector('[data-copy]');if(!b)return;"
    "b.addEventListener('click',function(){var t=new TextDecoder().decode("
    "Uint8Array.from(atob(document.getElementById('src').textContent.trim()),function(c){return c.charCodeAt(0)}));"
    "var done=function(){var l=b.textContent;b.textContent='copied ✓';"
    "setTimeout(function(){b.textContent=l},1500)};"
    "if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(t).then(done,function(){fb(t);done()})}"
    "else{fb(t);done()}});function fb(t){var a=document.createElement('textarea');a.value=t;"
    "a.style.position='fixed';a.style.opacity='0';document.body.appendChild(a);a.select();"
    "try{document.execCommand('copy')}catch(e){}document.body.removeChild(a)}})();</script>"
)


def sh(*a: str) -> bytes:
    return subprocess.check_output(a, cwd=args.repo)


def crumbs(path: str, is_file: bool) -> str:
    parts = path.split("/") if path else []
    dirparts = parts[:-1] if is_file else parts
    depth = len(dirparts)
    out = [f'<a href="{"../" * depth or "./"}index.html">{html.escape(APP)}</a>']
    for i, p in enumerate(dirparts):
        out.append(f'<a href="{"../" * (depth - 1 - i)}index.html">{html.escape(p)}</a>')
    if is_file and parts:
        out.append(html.escape(parts[-1]))
    return " / ".join(out)


def footer(depth: int) -> str:
    up = "../" * depth
    return (
        f'<p class="muted">static mirror of HEAD · <a href="{up}../index.html">about</a>'
        f' · clone: <code>git clone {html.escape(CLONE)}</code></p>'
    )


files = []  # (path, size)
for line in sh("git", "ls-tree", "-r", "--long", "HEAD").decode().splitlines():
    meta, path = line.split("\t", 1)
    size = meta.split()[3]
    files.append((path, int(size) if size.isdigit() else 0))

tracked = {p for p, _ in files}

dirs: dict[str, list] = {"": []}
for path, size in files:
    parts = path.split("/")
    for i in range(len(parts) - 1):
        d = "/".join(parts[: i + 1])
        parent = "/".join(parts[:i])
        dirs.setdefault(d, [])
        if ("d", parts[i]) not in [(t, n) for t, n, *_ in dirs[parent]]:
            dirs[parent].append(("d", parts[i]))
    dirs["/".join(parts[:-1])].append(("f", parts[-1], size, path))

for d, items in dirs.items():
    out = OUT / d / "index.html"
    out.parent.mkdir(parents=True, exist_ok=True)
    depth = d.count("/") + (1 if d else 0)
    rows = []
    if d:
        rows.append('<li><a href="../index.html">../</a></li>')
    readme = None
    for it in sorted(items, key=lambda x: (x[0] != "d", x[1].lower())):
        if it[0] == "d":
            rows.append(f'<li><a href="{html.escape(it[1])}/index.html">{html.escape(it[1])}/</a></li>')
        else:
            _, name, size, p = it
            rows.append(
                f'<li><a href="{html.escape(name)}.html">{html.escape(name)}'
                f'<span class="sz">{size:,} B</span></a></li>'
            )
            if name.lower() == "readme.md":
                readme = p
    inline_md = ""
    if readme and md_html is not None:
        text = sh("git", "show", f"HEAD:{readme}").decode("utf-8", "replace")
        inline_md = ('<div class="md" style="margin-top:16px">'
                     + fill_releases(relink_md(md_html(text), d, tracked), APP) + '</div>')
    title = html.escape(d or APP)
    out.write_text(
        f"{HEAD_TAG}<title>{title}</title><main>"
        f'<div class="crumb">{crumbs(d, False)}</div>'
        f'<h1>{title}/</h1><ul class="tree">{"".join(rows)}</ul>'
        f"{inline_md}{footer(depth)}</main>"
        f"{releases_script(depth) if 'data-rel=' in inline_md else ''}"
    )

for path, size in files:
    out = OUT / (path + ".html")
    out.parent.mkdir(parents=True, exist_ok=True)
    depth = path.count("/")
    blob = sh("git", "show", f"HEAD:{path}")
    name = path.rsplit("/", 1)[-1]
    if name in GENERATED:
        body = f'<p class="muted">generated file · {size:,} bytes · not rendered — see the git clone</p>'
    elif b"\0" in blob[:8000]:
        body = f'<p class="muted">binary file · {size:,} bytes</p>'
    elif size > MAX_RENDER:
        body = f'<p class="muted">file too large to render ({size:,} bytes) — see the git clone</p>'
    else:
        text = blob.decode("utf-8", "replace")
        tools = source_tools(text, name, depth)
        if name.lower().endswith((".md", ".markdown")) and md_html is not None:
            base = path.rsplit('/', 1)[0] if '/' in path else ''
            body = '<div class="md">' + fill_releases(relink_md(md_html(text), base, tracked), APP) + '</div>'
        elif highlight is not None:
            try:
                lexer = guess_lexer_for_filename(path, text)
            except ClassNotFound:
                # Pygments registers no lexer for .edn (Clojure data), so
                # deps.edn and friends would fall through to plain text
                lexer = ClojureLexer() if name.lower().endswith(".edn") else TextLexer()
            body = highlight(text, lexer, FORMATTER)
        else:
            body = f'<pre class="plain">{html.escape(text)}</pre>'
        body = tools + body
    out.write_text(
        f"{HEAD_TAG}<title>{html.escape(name)}</title><main>"
        f'<div class="crumb">{crumbs(path, True)}</div>{body}{footer(depth)}</main>'
        f"{releases_script(depth) if 'data-rel=' in body else ''}"
        f"{COPY_SCRIPT if 'data-copy=' in body else ''}"
    )

print(f"browse[{APP}]: {len(files)} files, {len(dirs)} dirs -> {OUT}")

static mirror of HEAD · about · clone: git clone https://ird.cod-sursa.ro/publish.git