corpora.py

"""Training corpora for the /merges/ page.

BERRIES is deliberately small and repetitive. That is the point: at this size
you can watch every merge and see why it happened. It is rigged only in the
sense that a demonstration is rigged — the algorithm is not told anything about
berries, and the same code run on your own text does the same thing.
"""

BERRIES = (
    "the strawberry and the raspberry and the blueberry and the blackberry\n"
    "are all berries. a strawberry is red. a raspberry is red. a blueberry is blue.\n"
    "the blackberry is black. berries grow on the berry bushes in the berry garden.\n"
    "she picked a strawberry, then another strawberry, then a raspberry."
)

# Article 1 of the UDHR — the same text the language table on /tokens/ uses.
UDHR_EN = (
    "All human beings are born free and equal in dignity and rights. They are "
    "endowed with reason and conscience and should act towards one another in a "
    "spirit of brotherhood."
)

CODE = (
    "def total(items):\n"
    "    return sum(item.price for item in items)\n"
    "def count(items):\n"
    "    return sum(1 for item in items)\n"
    "def names(items):\n"
    "    return [item.name for item in items]\n"
)

# Cases that break naive implementations.
EDGE = "🍓🍓🍓 café café éé \t\t\n\n  "
NO_REPEATS = "abcdefg hijklmn opqrstu"
JAPANESE = "これはトークンです。これはトークンです。これはテストです。"

# (name, text, merges to run)
CORPORA = [
    ("berries", BERRIES, 30),
    ("udhr-en", UDHR_EN, 20),
    ("code", CODE, 25),
    ("edge-unicode", EDGE, 10),
    ("no-repeats", NO_REPEATS, 10),
    ("japanese", JAPANESE, 12),
]

# The opening of A Tale of Two Cities (1859, public domain). Chosen because its
# parallel structure gives an n-gram unusually strong statistics — which is what
# makes greedy decoding loop so visibly.
TALE = (
    "It was the best of times, it was the worst of times, it was the age of "
    "wisdom, it was the age of foolishness, it was the epoch of belief, it was "
    "the epoch of incredulity, it was the season of Light, it was the season of "
    "Darkness, it was the spring of hope, it was the winter of despair, we had "
    "everything before us, we had nothing before us, we were all going direct to "
    "Heaven, we were all going direct the other way - in short, the period was so "
    "far like the present period, that some of its noisiest authorities insisted "
    "on its being received, for good or for evil, in the superlative degree of "
    "comparison only."
)