checkrequests.py

"""Every host a browser actually requested, parsed from Chrome's network log.

The site claims "no third-party requests" on every page. That was only checked
by grepping the generated HTML for foreign hostnames, which cannot see what a
script fetches at run time — and Cloudflare injects a script into every page.

A warning learned the hard way: do NOT grep URLs out of the netlog. The file
contains Chrome's own configuration — the HSTS preload list, safe-browsing and
autofill tables — so a regex over it "finds" youtube.com and play.google.com on
a page that never touched them. That would have been a confident, wrong,
alarming claim. This parses real request events instead.

Capture and analysis are separate: only shot.sh may reach the network and start
chrome, so run it per page first, then

    python3 checkrequests.py /tmp/claude-996/net-*.json
"""

import json
import os
import re
import sys

OWN = {"sweedworks.com", "www.sweedworks.com"}

# Hosts Chrome contacts on its own account at startup. Verified by capturing a
# load of /robots.txt — a plain text file with no HTML, CSS or scripts — and
# seeing exactly these. They would appear on any site in the world. Listed
# rather than filtered silently, and deliberately narrow: a real request to a
# Google host from page code would still be reported.
BROWSER_STARTUP = {
    "accounts.google.com",
    "www.google.com",
    "clients2.google.com",
    "content-autofill.googleapis.com",
}
HOST_RE = re.compile(r"^https?://([^/:]+)")

# Event types that mean a request was actually started.
REQUEST_EVENTS = {"URL_REQUEST_START_JOB", "HTTP_STREAM_JOB_CONTROLLER_BOUND"}


def analyse(path):
    with open(path, encoding="utf-8", errors="replace") as fh:
        raw = fh.read()

    # Chrome may not close the JSON if it was killed; salvage what parses.
    try:
        log = json.loads(raw)
    except ValueError:
        cut = raw.rfind("},")
        log = json.loads(raw[:cut + 1] + "]}")

    types = log.get("constants", {}).get("logEventTypes", {})
    wanted = {tid for name, tid in types.items() if name in REQUEST_EVENTS}
    if not wanted:
        raise RuntimeError(f"{path}: no request event types in constants")

    requested = []
    for ev in log.get("events", []):
        if ev.get("type") not in wanted:
            continue
        url = (ev.get("params") or {}).get("url")
        if url:
            requested.append(url)

    own, foreign, browser = set(), {}, {}
    for url in requested:
        m = HOST_RE.match(url)
        if not m:
            continue
        host = m.group(1).lower()
        if host in OWN:
            own.add(url.split("?")[0])
        elif host in BROWSER_STARTUP:
            browser[host] = browser.get(host, 0) + 1
        else:
            foreign[host] = foreign.get(host, 0) + 1
    return own, foreign, browser, len(requested)


def main():
    logs = sys.argv[1:]
    if not logs:
        print(__doc__)
        return 2

    problems = []
    for path in logs:
        if not os.path.exists(path):
            problems.append(f"{path}: missing — was shot.sh run for it?")
            continue
        own, foreign, browser, total = analyse(path)
        print(f"{os.path.basename(path)}  ({total} requests started)")
        for u in sorted(own):
            tag = "cloudflare" if "/cdn-cgi/" in u else "own"
            print(f"    {tag:<10} {u}")
        for h, n in sorted(browser.items()):
            print(f"    browser    {h} ({n})")
        for h, n in sorted(foreign.items()):
            print(f"    FOREIGN  {h} ({n})")
            problems.append(f"{os.path.basename(path)}: requested {h}")
        print()

    if problems:
        print(f"{len(problems)} problem(s):")
        for p in problems:
            print(f"  - {p}")
        return 1
    print("Every request from page code went to sweedworks.com. The only other "
          "traffic is Chrome's own startup calls, which happen on any site.")
    return 0


if __name__ == "__main__":
    sys.exit(main())