checkassets.py

"""Compare every served asset, byte for byte, against the file I generated.

verify.py checks the tokenizer bundles on disk. checklive.py checks that the
HTML arrives unmodified. Nothing checked the scripts and stylesheets in
transit — and Cloudflare provably rewrites HTML on the way out, so "it is
correct on disk" is not the same claim as "it is correct when it reaches a
reader". Cloudflare has features that rewrite JavaScript (Rocket Loader, and
minification in older plans); if one were ever switched on, every gradient
check and tokenizer comparison on this site would be verifying a file nobody
receives.

    python3 checkassets.py
"""

import hashlib
import os
import re
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, ".."))
BASE = "https://sweedworks.com"

PAGES = ["index.html", "tokens/index.html", "vocabulary/index.html",
         "predict/index.html", "cost/index.html", "learn/index.html",
         "attention/index.html", "about/index.html", "changes/index.html"]

# Assets no page links directly but which readers still receive.
EXTRA = ["/vendor/gpt-tokenizer/o200k_base.js",
         "/vendor/gpt-tokenizer/cl100k_base.js",
         "/feed.xml", "/sitemap.xml", "/robots.txt", "/favicon.svg",
         "/favicon.ico", "/404.html"]

ASSET_RE = re.compile(r'(?:href|src)="(/[^"]+\.(?:js|css|svg|ico|png|xml|txt))')


def local_path(url):
    return os.path.join(ROOT, url.split("?")[0].lstrip("/"))


def fetch(url):
    out = subprocess.run(["curl", "-sS", "--compressed", "-o", "-", BASE + url],
                         capture_output=True, timeout=120)
    return out.stdout


def main():
    wanted = set(EXTRA)
    for page in PAGES:
        src = open(os.path.join(ROOT, page), encoding="utf-8").read()
        for m in ASSET_RE.finditer(src):
            wanted.add(m.group(1).split("?")[0])

    problems, notes, checked = [], [], 0
    for url in sorted(wanted):
        path = local_path(url)
        if not os.path.isfile(path):
            problems.append(f"{url}: referenced but not on disk")
            continue
        mine = open(path, "rb").read()
        served = fetch(url)
        checked += 1
        if served == mine:
            continue

        # HTML is rewritten in transit by Cloudflare, which /about/ discloses.
        # Accept that difference only if it really is that injection: the
        # marker must be present, and everything else must survive line for
        # line. Anything more is a finding.
        if url.endswith(".html") or url.endswith("/"):
            text = served.decode("utf-8", "replace")
            original = mine.decode("utf-8", "replace")
            residue = text
            for line in original.splitlines():
                residue = residue.replace(line, "", 1)
            residue = residue.strip()
            if "__CF$cv$params" in residue and len(residue) < 2000:
                notes.append(f"{url}: {len(residue):,} bytes injected in "
                             f"transit (Cloudflare, disclosed)")
                continue
            problems.append(f"{url}: HTML altered in transit in a way that is "
                            f"not the disclosed injection ({len(residue):,} "
                            f"bytes: {residue[:120]!r})")
            continue
        # Not identical. Say how, because "differs" is not actionable.
        note = (f"{url}: served {len(served):,} bytes, generated "
                f"{len(mine):,} bytes")
        if len(served) == 0:
            note += " — empty response"
        elif mine in served:
            note += " — something was appended or prepended in transit"
        else:
            note += (f" — content differs (sha256 "
                     f"{hashlib.sha256(served).hexdigest()[:12]} vs "
                     f"{hashlib.sha256(mine).hexdigest()[:12]})")
        problems.append(note)

    print(f"compared {checked} assets byte for byte")
    for n in notes:
        print(f"  note  {n}")
    print()
    if problems:
        print(f"{len(problems)} problem(s):")
        for p in problems:
            print(f"  - {p}")
        return 1
    print("Every script, stylesheet and data file arrives exactly as generated. "
          "Only HTML is rewritten in transit, which /about/ discloses.")
    return 0


if __name__ == "__main__":
    sys.exit(main())