Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Byte-level Byte Pair Encoding (BPE) tokenizer trained on raw text with token-to-id mapping.
"""Part 4 · From text to GPT (1/2): a byte-level BPE tokenizer.
Start from the 256 possible bytes (so any text, in any language, can be encoded), then repeatedly merge the
most frequent adjacent pair into a new token. Frequent words become single tokens; rare ones stay as pieces.
"""
from __future__ import annotations
import json
import re
from collections import Counter
# Pre-split into pieces: a word keeps its leading space (" the"), like GPT tokenizers; merges never cross pieces.
PIECE = re.compile(r" ?[^ ]+| +")
def _merge(ids: list, pair: tuple, new: int) -> list:
out, i = [], 0
while i < len(ids):
if i + 1 < len(ids) and ids[i] == pair[0] and ids[i + 1] == pair[1]:
out.append(new)
i += 2
else:
out.append(ids[i])
i += 1
return out
class BPETokenizer:
def __init__(self, merges: list | None = None):
self.merges = [tuple(m) for m in (merges or [])]
self._rebuild()
def _rebuild(self):
self.vocab = {i: bytes([i]) for i in range(256)}
self.rank = {}
for i, (a, b) in enumerate(self.merges):
new = 256 + i
self.vocab[new] = self.vocab[a] + self.vocab[b]
self.rank[(a, b)] = i
@property
def vocab_size(self) -> int:
return 256 + len(self.merges)
@classmethod
def train(cls, text: str, vocab_size: int = 512) -> "BPETokenizer":
# Count each distinct piece once, with its frequency: far faster than scanning the raw text every merge.
words = Counter(PIECE.findall(text))
seqs = {w: list(w.encode("utf-8")) for w in words}
merges = []
for i in range(vocab_size - 256):
pairs = Counter()
for w, ids in seqs.items():
f = words[w]
for p in zip(ids, ids[1:]):
pairs[p] += f
if not pairs:
break
best = max(pairs, key=pairs.get)
new = 256 + i
merges.append(best)
for w in seqs:
if len(seqs[w]) > 1:
seqs[w] = _merge(seqs[w], best, new)
return cls(merges)
def encode(self, text: str) -> list:
out = []
for piece in PIECE.findall(text):
ids = list(piece.encode("utf-8"))
while len(ids) > 1:
# Apply the earliest-learned merge first, exactly as in training.
pair = min(zip(ids, ids[1:]), key=lambda p: self.rank.get(p, 1 << 30))
if pair not in self.rank:
break
ids = _merge(ids, pair, 256 + self.rank[pair])
out.extend(ids)
return out
def decode(self, ids: list) -> str:
return b"".join(self.vocab[i] for i in ids).decode("utf-8", errors="replace")
def save(self, path) -> None:
with open(path, "w") as f:
json.dump({"merges": self.merges}, f)
@classmethod
def load(cls, path) -> "BPETokenizer":
with open(path) as f:
return cls(json.load(f)["merges"])