/* ════════════════════════════════════════════════════════════════════
   L8 OS · NANO · TOKENIZER  ·  byte-pair encoding, trained on your text
   ────────────────────────────────────────────────────────────────────
   Starts at raw bytes, so there is nothing it cannot represent — no
   unknown token, no dropped character, any language and any binary
   sludge round-trips exactly. Then it repeatedly merges the most common
   adjacent pair until the vocabulary is the size you asked for.

   Trained on your own corpus this learns YOUR text: `function`, `var`,
   `L8`, `DIVA`, `.split(` and `://` become single tokens because that
   is what is actually frequent in these files. A generic tokenizer
   would spend three tokens where this spends one, and for a model this
   small the context window is the scarcest thing there is.

   Round-tripping is not assumed — train.cjs asserts decode(encode(x))
   is byte-identical to x before a single training step runs.
   ════════════════════════════════════════════════════════════════════ */
'use strict';

function toBytes(str) {
  if (typeof Buffer !== 'undefined') return Uint8Array.from(Buffer.from(str, 'utf8'));
  return new TextEncoder().encode(str);
}
function fromBytes(bytes) {
  var u8 = bytes instanceof Uint8Array ? bytes : Uint8Array.from(bytes);
  if (typeof Buffer !== 'undefined') return Buffer.from(u8).toString('utf8');
  return new TextDecoder('utf8', { fatal: false }).decode(u8);
}

/* ── train ──────────────────────────────────────────────────────────
   Straightforward BPE. The corpus is held as one flat array of ids and
   each round rewrites it in place, which is O(n) per merge — slow in
   theory, fine here, and simple enough to be obviously correct. */
function train(text, vocabSize, onProgress) {
  var ids = Array.from(toBytes(text));
  var merges = [];                             /* [a, b] in merge order */
  var next = 256;
  var target = Math.max(256, vocabSize | 0);

  while (next < target) {
    var counts = new Map(), i;
    for (i = 0; i + 1 < ids.length; i++) {
      var key = ids[i] * 100000 + ids[i + 1];
      counts.set(key, (counts.get(key) || 0) + 1);
    }
    var bestKey = -1, bestN = 1;
    counts.forEach(function (n, key) { if (n > bestN) { bestN = n; bestKey = key; } });
    if (bestKey < 0) break;                    /* nothing repeats — done */

    var a = Math.floor(bestKey / 100000), b = bestKey % 100000;
    var out = [], j = 0;
    while (j < ids.length) {
      if (j + 1 < ids.length && ids[j] === a && ids[j + 1] === b) { out.push(next); j += 2; }
      else { out.push(ids[j]); j++; }
    }
    ids = out;
    merges.push([a, b]);
    next++;
    if (onProgress && next % 32 === 0) onProgress(next, target, bestN, ids.length);
  }
  return pack(merges);
}

/* the vocabulary as byte strings, so decode is a concatenation */
function pack(merges) {
  var vocab = new Array(256 + merges.length);
  for (var i = 0; i < 256; i++) vocab[i] = [i];
  merges.forEach(function (m, k) { vocab[256 + k] = vocab[m[0]].concat(vocab[m[1]]); });
  var rank = new Map();
  merges.forEach(function (m, k) { rank.set(m[0] + ',' + m[1], k); });
  return { merges: merges, vocab: vocab, rank: rank, size: vocab.length };
}

/* ── encode · apply merges in the order they were learned ──────────── */
function encode(tk, text) {
  var ids = Array.from(toBytes(text));
  if (!ids.length) return new Int32Array(0);
  for (;;) {
    var bestRank = Infinity, bestAt = -1, i;
    for (i = 0; i + 1 < ids.length; i++) {
      var r = tk.rank.get(ids[i] + ',' + ids[i + 1]);
      if (r !== undefined && r < bestRank) { bestRank = r; bestAt = i; }
    }
    if (bestAt < 0) break;
    var id = 256 + bestRank, out = [], j = 0;
    var a = tk.merges[bestRank][0], b = tk.merges[bestRank][1];
    while (j < ids.length) {
      if (j + 1 < ids.length && ids[j] === a && ids[j + 1] === b) { out.push(id); j += 2; }
      else { out.push(ids[j]); j++; }
    }
    ids = out;
  }
  return Int32Array.from(ids);
}

function decode(tk, ids) {
  var bytes = [];
  for (var i = 0; i < ids.length; i++) {
    var v = tk.vocab[ids[i]];
    if (v) for (var j = 0; j < v.length; j++) bytes.push(v[j]);
  }
  return fromBytes(bytes);
}

/* a single token as text, for showing the vocabulary to a person */
function piece(tk, id) {
  var v = tk.vocab[id];
  return v ? fromBytes(v) : '';
}

function toJSON(tk) { return { format: 'l8-bpe-1', merges: tk.merges }; }
function fromJSON(j) { return pack(j.merges || []); }

var TOK = { train: train, encode: encode, decode: decode, piece: piece,
  toJSON: toJSON, fromJSON: fromJSON, pack: pack, toBytes: toBytes, fromBytes: fromBytes };

if (typeof module !== 'undefined' && module.exports) module.exports = TOK;
if (typeof globalThis !== 'undefined') globalThis.L8Tok = TOK;