cjsload.py

"""Load gpt-tokenizer's CommonJS build under QuickJS via a minimal require()
shim. Used to independently verify the vendored dist/ bundles — if two
different code paths agree on token IDs, the vendored bundle is trustworthy."""

import json
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "pylib"))
import quickjs  # noqa: E402

from tokenlib import POLYFILL  # noqa: E402

CJS = os.path.join(HERE, "gpt-tokenizer-cjs")

REQUIRE_SHIM = r"""
var __cache = {};
function __dirname_of(p){ return p.substring(0, p.lastIndexOf('/')); }
function __normalize(p){
  var parts = p.split('/'), out = [];
  for (var i = 0; i < parts.length; i++) {
    if (parts[i] === '.' || parts[i] === '') continue;
    if (parts[i] === '..') out.pop(); else out.push(parts[i]);
  }
  return '/' + out.join('/');
}
function __resolve(base, req){
  var p = req.charAt(0) === '.' ? __normalize(base + '/' + req) : __normalize(req);
  var cands = [p, p + '.js', p + '/index.js'];
  for (var i = 0; i < cands.length; i++) if (__exists(cands[i])) return cands[i];
  throw new Error('cannot resolve ' + req + ' from ' + base);
}
function __require(base, req){
  var path = __resolve(base, req);
  if (__cache[path]) return __cache[path].exports;
  var mod = { exports: {} };
  __cache[path] = mod;
  var src = __readFile(path);
  var dir = __dirname_of(path);
  var fn = new Function('exports', 'require', 'module', '__filename', '__dirname', src);
  fn(mod.exports, function(r){ return __require(dir, r); }, mod, path, dir);
  return mod.exports;
}
"""


class Reference:
    """The trustworthy path: gpt-tokenizer's own CJS build, loaded from source."""

    def __init__(self, encoding="o200k_base"):
        self.encoding = encoding
        ctx = quickjs.Context()
        ctx.set_memory_limit(1 << 30)
        ctx.set_max_stack_size(1 << 22)
        ctx.add_callable("__readFile", lambda p: open(p, encoding="utf-8").read())
        ctx.add_callable("__exists", lambda p: os.path.isfile(p))
        ctx.eval(POLYFILL)
        ctx.eval(REQUIRE_SHIM)
        ctx.set("_cjs", CJS)
        ctx.set("_enc", encoding)
        ctx.eval("var M = __require(_cjs, './encoding/' + _enc + '.js');")
        ctx.eval("""
        function tokenize(s){
          var ids = M.encode(s), out = [];
          for (var i = 0; i < ids.length; i++) out.push([ids[i], M.decode([ids[i]])]);
          return JSON.stringify(out);
        }
        """)
        self.ctx = ctx

    def ids(self, text):
        self.ctx.set("_s", text)
        return json.loads(self.ctx.eval("JSON.stringify(M.encode(_s))"))

    def tokens(self, text):
        self.ctx.set("_s", text)
        return [{"id": i, "text": t} for i, t in json.loads(self.ctx.eval("tokenize(_s)"))]

    def count(self, text):
        return len(self.ids(text))


if __name__ == "__main__":
    for enc in ("cl100k_base", "o200k_base"):
        r = Reference(enc)
        print(f"  {enc}: 'hello world' -> {r.ids('hello world')}")