verify.py

"""Assert the browser bundle we ship agrees with gpt-tokenizer's reference build.

This exists because gpt-tokenizer 3.4.0's dist/cl100k_base.js is mislabeled: it
declares GPTTokenizer_cl100k_base but emits o200k token IDs. A filename is not
evidence. Run this before every deploy.
"""

import sys
import unicodedata

from cjsload import Reference
from tokenlib import Tokenizer

CORPUS = [
    "strawberry", " strawberry", "strawberry ", "Strawberry", "STRAWBERRY",
    "hello world", "Hello", " Hello", "", " ", "  ", "\n", "\n\n", "\t",
    "1234567890", "1,234,567,890", "3.14159265", "127.0.0.1", "2024", "20240811",
    "def total(items):\n    return sum(i.price for i in items)\n",
    "SELECT * FROM users WHERE id = 42;",
    "https://sweedworks.com/tokens/",
    "こんにちは世界", "の", "人人生而自由", "안녕하세요", "Привет, мир",
    "مرحبا بالعالم", "नमस्ते दुनिया", "Ω≈ç√∫˜µ≤≥÷",
    "café", "café",            # precomposed vs combining — different tokens
    "🍓", "👩‍👩‍👧‍👦", "é́́",
    "a" * 200, "🍓" * 40,
    "The tokenizer does not know what a word is.",
]


def check_encoding(enc):
    """Every bundle we serve must agree with the reference, token for token."""
    shipped = Tokenizer(enc)       # the bundle browsers actually download
    reference = Reference(enc)     # the package's own CJS source

    failures = []
    for text in CORPUS:
        a = [t["id"] for t in shipped.tokens(text)]
        b = reference.ids(text)
        if a != b:
            failures.append((text, a, b))

    if failures:
        print(f"FAIL  {enc}: {len(failures)} of {len(CORPUS)} strings mismatch")
        for text, a, b in failures[:6]:
            print(f"  {text!r}\n    shipped   {a[:8]}\n    reference {b[:8]}")
        return 1
    print(f"PASS  {enc}: {len(CORPUS)} strings — shipped bundle matches reference")

    bad = []
    for text in CORPUS:
        shipped.ctx.set("_s", text)
        if shipped.ctx.eval("T.decode(T.encode(_s))") != text:
            bad.append(text)
    if bad:
        print(f"FAIL  {enc}: round trip changed {len(bad)} string(s): {bad[:3]}")
        return 1
    print(f"PASS  {enc}: round trip decode(encode(x)) == x")
    return 0


def main():
    bad = 0
    for enc in ("o200k_base", "cl100k_base"):
        bad += check_encoding(enc)
    if bad:
        return 1

    # The two encodings must not be the same thing wearing different names.
    # This is the exact check that caught upstream's mislabeled bundle.
    a = Tokenizer("o200k_base")
    b = Tokenizer("cl100k_base")
    if a.count("hello world") == b.count("hello world") and \
       [t["id"] for t in a.tokens("hello world")] == \
       [t["id"] for t in b.tokens("hello world")]:
        print("FAIL  the two shipped bundles produce identical IDs — "
              "one of them is mislabeled")
        return 1
    print("PASS  the two shipped bundles are genuinely different vocabularies")

    shipped = a
    reference = Reference("o200k_base")

    # Sanity: this encoding must actually be o200k, not something wearing its name.
    # "hello world" is [15339,1917] under cl100k and [24912,2375] under o200k, so
    # it distinguishes the two. The strawberry split is asserted on decoded text
    # rather than raw ids — that's the claim the site actually makes.
    if reference.ids("hello world") != [24912, 2375]:
        print(f"FAIL  identity: 'hello world' -> {reference.ids('hello world')}")
        return 1
    split = [t["text"] for t in shipped.tokens("strawberry")]
    if split != ["st", "raw", "berry"]:
        print(f"FAIL  identity: 'strawberry' splits as {split}, expected st/raw/berry")
        return 1
    if shipped.count(" strawberry") != 1:
        print("FAIL  identity: ' strawberry' should be a single token")
        return 1
    print("PASS  identity — encoding is genuinely o200k_base")
    return 0


if __name__ == "__main__":
    sys.exit(main())