corpora.py
The training texts used throughout
61 lines. This is the file the build actually runs, copied verbatim at build time.
- 1
"""Training corpora for the /merges/ page. - 2
- 3
BERRIES is deliberately small and repetitive. That is the point: at this size - 4
you can watch every merge and see why it happened. It is rigged only in the - 5
sense that a demonstration is rigged — the algorithm is not told anything about - 6
berries, and the same code run on your own text does the same thing. - 7
""" - 8
- 9
BERRIES = ( - 10
"the strawberry and the raspberry and the blueberry and the blackberry\n" - 11
"are all berries. a strawberry is red. a raspberry is red. a blueberry is blue.\n" - 12
"the blackberry is black. berries grow on the berry bushes in the berry garden.\n" - 13
"she picked a strawberry, then another strawberry, then a raspberry." - 14
) - 15
- 16
# Article 1 of the UDHR — the same text the language table on /tokens/ uses. - 17
UDHR_EN = ( - 18
"All human beings are born free and equal in dignity and rights. They are " - 19
"endowed with reason and conscience and should act towards one another in a " - 20
"spirit of brotherhood." - 21
) - 22
- 23
CODE = ( - 24
"def total(items):\n" - 25
" return sum(item.price for item in items)\n" - 26
"def count(items):\n" - 27
" return sum(1 for item in items)\n" - 28
"def names(items):\n" - 29
" return [item.name for item in items]\n" - 30
) - 31
- 32
# Cases that break naive implementations. - 33
EDGE = "🍓🍓🍓 café café éé \t\t\n\n " - 34
NO_REPEATS = "abcdefg hijklmn opqrstu" - 35
JAPANESE = "これはトークンです。これはトークンです。これはテストです。" - 36
- 37
# (name, text, merges to run) - 38
CORPORA = [ - 39
("berries", BERRIES, 30), - 40
("udhr-en", UDHR_EN, 20), - 41
("code", CODE, 25), - 42
("edge-unicode", EDGE, 10), - 43
("no-repeats", NO_REPEATS, 10), - 44
("japanese", JAPANESE, 12), - 45
] - 46
- 47
# The opening of A Tale of Two Cities (1859, public domain). Chosen because its - 48
# parallel structure gives an n-gram unusually strong statistics — which is what - 49
# makes greedy decoding loop so visibly. - 50
TALE = ( - 51
"It was the best of times, it was the worst of times, it was the age of " - 52
"wisdom, it was the age of foolishness, it was the epoch of belief, it was " - 53
"the epoch of incredulity, it was the season of Light, it was the season of " - 54
"Darkness, it was the spring of hope, it was the winter of despair, we had " - 55
"everything before us, we had nothing before us, we were all going direct to " - 56
"Heaven, we were all going direct the other way - in short, the period was so " - 57
"far like the present period, that some of its noisiest authorities insisted " - 58
"on its being received, for good or for evil, in the superlative degree of " - 59
"comparison only." - 60
)