precompute.py

"""Generate tokens/data.json — every number shown on the page is computed here,
never hand-written, so the prose can't drift from the tokenizer."""

import json
import os

from cjsload import Reference

OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tokens", "data.json")

# Article 1 of the Universal Declaration of Human Rights, in the UN's official
# translations. Chosen because it's the same meaning in every row — which is the
# whole point: identical content, wildly different token cost.
UDHR = [
    ("English", "en", "Latin",
     "All human beings are born free and equal in dignity and rights. They are "
     "endowed with reason and conscience and should act towards one another in a "
     "spirit of brotherhood."),
    ("Spanish", "es", "Latin",
     "Todos los seres humanos nacen libres e iguales en dignidad y derechos y, "
     "dotados como están de razón y conciencia, deben comportarse fraternalmente "
     "los unos con los otros."),
    ("French", "fr", "Latin",
     "Tous les êtres humains naissent libres et égaux en dignité et en droits. "
     "Ils sont doués de raison et de conscience et doivent agir les uns envers "
     "les autres dans un esprit de fraternité."),
    ("German", "de", "Latin",
     "Alle Menschen sind frei und gleich an Würde und Rechten geboren. Sie sind "
     "mit Vernunft und Gewissen begabt und sollen einander im Geiste der "
     "Brüderlichkeit begegnen."),
    ("Portuguese", "pt", "Latin",
     "Todos os seres humanos nascem livres e iguais em dignidade e direitos. São "
     "dotados de razão e consciência e devem agir uns para com os outros em "
     "espírito de fraternidade."),
    ("Russian", "ru", "Cyrillic",
     "Все люди рождаются свободными и равными в своем достоинстве и правах. Они "
     "наделены разумом и совестью и должны поступать в отношении друг друга в "
     "духе братства."),
    ("Chinese", "zh", "Han",
     "人人生而自由，在尊严和权利上一律平等。他们赋有理性和良心，并应以兄弟关系的精神相对待。"),
    ("Japanese", "ja", "Japanese",
     "すべての人間は、生れながらにして自由であり、かつ、尊厳と権利とについて平等である。"
     "人間は、理性と良心とを授けられており、互いに同胞の精神をもって行動しなければならない。"),
    ("Korean", "ko", "Hangul",
     "모든 인간은 태어날 때부터 자유로우며 그 존엄과 권리에 있어 동등하다. 인간은 천부적으로 "
     "이성과 양심을 부여받았으며 서로 형제애의 정신으로 행동하여야 한다."),
    ("Arabic", "ar", "Arabic",
     "يولد جميع الناس أحراراً متساوين في الكرامة والحقوق. وقد وهبوا عقلاً وضميراً "
     "وعليهم أن يعامل بعضهم بعضاً بروح الإخاء."),
    ("Hindi", "hi", "Devanagari",
     "सभी मनुष्यों को गौरव और अधिकारों के मामले में जन्मजात स्वतन्त्रता और समानता प्राप्त है। "
     "उन्हें बुद्धि और अन्तरात्मा की देन प्राप्त है और परस्पर उन्हें भाईचारे के भाव से बर्ताव करना चाहिए।"),
]

COUNTING_WORDS = ["strawberry", "raspberry", "bookkeeper", "Mississippi", "unsuccessfully"]

NUMBERS = ["1234567890", "1,234,567,890", "3.14159265", "2024", "20240811", "127.0.0.1"]

WHITESPACE = ["strawberry", " strawberry", "strawberry ", "Strawberry", "STRAWBERRY", "  strawberry"]

CODE = 'def total(items):\n    return sum(i.price for i in items)\n'

PROSE = (
    "The tokenizer does not know what a word is. It knows which byte sequences "
    "showed up together often enough during training to deserve their own number."
)


def letter_story(tok, word, letter):
    """How the model sees a word it's being asked to spell."""
    toks = tok.tokens(word)
    return {
        "word": word,
        "letter": letter,
        "true_count": word.lower().count(letter),
        "tokens": toks,
        "token_count": len(toks),
    }


def main():
    o200k = Reference("o200k_base")
    cl100k = Reference("cl100k_base")

    data = {
        "generated_note": "All figures computed by .build/precompute.py from "
                          "gpt-tokenizer's reference (CJS) build, verified against "
                          "the shipped browser bundle by .build/verify.py.",
        "encodings": {
            "o200k_base": {"vocab": int(o200k.ctx.eval("M.vocabularySize")),
                           "used_by": "GPT-4o and o-series models"},
            "cl100k_base": {"vocab": int(cl100k.ctx.eval("M.vocabularySize")),
                            "used_by": "GPT-4 and GPT-3.5-turbo"},
        },
        "counting": [letter_story(o200k, w, l) for w, l in
                     zip(COUNTING_WORDS, ["r", "r", "k", "s", "s"])],
        "numbers": [{"text": n, "tokens": o200k.tokens(n)} for n in NUMBERS],
        "whitespace": [{"text": w, "tokens": o200k.tokens(w)} for w in WHITESPACE],
        "code": {"text": CODE, "tokens": o200k.tokens(CODE)},
        "prose": {"text": PROSE, "tokens": o200k.tokens(PROSE)},
        "languages": [],
    }

    english_tokens = None
    for name, code, script, text in UDHR:
        n_o = o200k.count(text)
        n_c = cl100k.count(text)
        if english_tokens is None:
            english_tokens = n_o
        data["languages"].append({
            "name": name, "code": code, "script": script, "text": text,
            "chars": len(text),
            "o200k": n_o,
            "cl100k": n_c,
            "chars_per_token": round(len(text) / n_o, 2),
            "vs_english": round(n_o / english_tokens, 2),
        })

    # Same text, two model generations. Newer vocabulary, fewer tokens for the
    # same meaning — most dramatically outside English.
    data["encoding_shift"] = [
        {"label": name, "text": text,
         "cl100k": cl100k.count(text), "o200k": o200k.count(text)}
        for name, text in [
            ("English prose", PROSE),
            ("Python", CODE),
            ("Japanese", UDHR[7][3]),
            ("Hindi", UDHR[10][3]),
            ("Arabic", UDHR[9][3]),
        ]
    ]

    with open(OUT, "w", encoding="utf-8") as fh:
        json.dump(data, fh, ensure_ascii=False, indent=1)

    print(f"wrote {OUT}")
    print(f"  o200k_base vocab: {data['encodings']['o200k_base']['vocab']:,}")
    print(f"  cl100k_base vocab: {data['encodings']['cl100k_base']['vocab']:,}")
    print("\n  language          chars  o200k  cl100k  chars/tok  vs EN")
    for r in data["languages"]:
        print(f"  {r['name']:<16}{r['chars']:>6}{r['o200k']:>7}{r['cl100k']:>8}"
              f"{r['chars_per_token']:>11}{r['vs_english']:>7}x")


if __name__ == "__main__":
    main()