Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Runnable checkpoint: trains a 940K parameter TinyGPT on curriculum text and samples generated tokens.
"""Part 4 · From text to GPT: train a BPE tokenizer and TinyGPT on the lessons. Saves runs/tinygpt.pt.
python3 scripts/part4_gpt.py --steps 800 (about a minute and a half on a laptop CPU)
"""
import argparse
import math
import common
import numpy as np
import torch
from atlas.data import corpus_text, load_lessons, train_val_split
from atlas.gpt import GPTConfig, TinyGPT, train_lm, unigram_loss
from atlas.tokenizer import BPETokenizer
ap = argparse.ArgumentParser()
ap.add_argument("--steps", type=int, default=800)
ap.add_argument("--eval-every", type=int, default=100)
ap.add_argument("--vocab", type=int, default=1024)
args = ap.parse_args()
common.banner("Part 4 · From text to GPT")
torch.manual_seed(0)
text = corpus_text(load_lessons())
tok = BPETokenizer.train(text, args.vocab)
ids = np.array(tok.encode(text))
assert tok.decode(ids.tolist()) == text # lossless round trip
train, val = train_val_split(ids)
print(f"BPE vocab {tok.vocab_size} · {len(ids):,} tokens ({len(text) / len(ids):.2f} chars/token) · train {len(train):,} · val {len(val):,}")
print("tokens of a sentence:", [tok.decode([i]) for i in tok.encode("Atlas answers questions about Python")])
model = TinyGPT(GPTConfig(vocab_size=tok.vocab_size))
print(f"TinyGPT: {model.num_params():,} parameters")
print(f"bars to beat: uniform guess ln(V) = {math.log(tok.vocab_size):.3f} · unigram frequencies = {unigram_loss(train, tok.vocab_size):.3f}")
hist = train_lm(model, train, val, steps=args.steps, eval_every=args.eval_every)
best_val = min(v for _, _, v in hist)
prompt = "The GIL"
out = model.generate(torch.tensor([tok.encode(prompt)]), 60, temperature=0.8, top_k=40)
print(f"\nsample (temperature 0.8): {tok.decode(out[0].tolist())!r}")
print(f"final train loss {hist[-1][1]:.3f} · best val loss {best_val:.3f} (val perplexity {math.exp(best_val):.1f}); those weights are kept")
print("Reading it: val loss is far below the unigram bar, so TinyGPT learned real structure. The gap between")
print("train and val, and val rising late in training, is memorization: 70K tokens is tiny. We keep the best")
print("checkpoint (early stopping); more data, not more steps, is the real fix.")
torch.save({"state": model.state_dict(), "config": model.config.__dict__}, common.RUNS / "tinygpt.pt")
tok.save(common.RUNS / "tokenizer.json")
print(f"\nCheckpoint: TinyGPT saved to {common.RUNS / 'tinygpt.pt'}")