Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Complete causal transformer architecture: multi-head self-attention, rotary embeddings, LayerNorm, and MLP blocks.
"""Part 4 · From text to GPT (2/2): TinyGPT, a decoder-only transformer.
token ids (B, T) -> embeddings (B, T, d) -> N x [pre-norm causal self-attention + MLP, each with a residual]
-> final norm -> logits (B, T, V). The output layer reuses the embedding matrix (weight tying).
"""
from __future__ import annotations
import math
from dataclasses import dataclass
import torch
from torch import nn
from torch.nn import functional as F
@dataclass
class GPTConfig:
vocab_size: int = 512
block_size: int = 128
n_layer: int = 4
n_head: int = 4
d_model: int = 128
dropout: float = 0.1
class CausalSelfAttention(nn.Module):
def __init__(self, c: GPTConfig):
super().__init__()
assert c.d_model % c.n_head == 0
self.n_head, self.dropout = c.n_head, c.dropout
self.qkv = nn.Linear(c.d_model, 3 * c.d_model)
self.proj = nn.Linear(c.d_model, c.d_model)
def forward(self, x):
B, T, C = x.shape
q, k, v = self.qkv(x).split(C, dim=2)
# (B, T, C) -> (B, heads, T, head_dim)
q, k, v = (t.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) for t in (q, k, v))
# is_causal=True: token t only attends to tokens <= t (the future would leak the answer).
y = F.scaled_dot_product_attention(q, k, v, is_causal=True, dropout_p=self.dropout if self.training else 0.0)
return self.proj(y.transpose(1, 2).contiguous().view(B, T, C))
class Block(nn.Module):
def __init__(self, c: GPTConfig):
super().__init__()
self.ln1, self.attn = nn.LayerNorm(c.d_model), CausalSelfAttention(c)
self.ln2 = nn.LayerNorm(c.d_model)
self.mlp = nn.Sequential(nn.Linear(c.d_model, 4 * c.d_model), nn.GELU(), nn.Linear(4 * c.d_model, c.d_model), nn.Dropout(c.dropout))
def forward(self, x):
x = x + self.attn(self.ln1(x)) # residual: the skip path keeps gradients flowing
return x + self.mlp(self.ln2(x))
class TinyGPT(nn.Module):
def __init__(self, c: GPTConfig):
super().__init__()
self.config = c
self.tok = nn.Embedding(c.vocab_size, c.d_model)
self.pos = nn.Embedding(c.block_size, c.d_model)
self.drop = nn.Dropout(c.dropout)
self.blocks = nn.ModuleList(Block(c) for _ in range(c.n_layer))
self.ln_f = nn.LayerNorm(c.d_model)
self.head = nn.Linear(c.d_model, c.vocab_size, bias=False)
self.head.weight = self.tok.weight # weight tying: one (V, d) matrix at both ends
self.apply(self._init)
@staticmethod
def _init(m):
# Small init so the first loss is close to ln(V), a uniform guess, not confidently wrong.
if isinstance(m, (nn.Linear, nn.Embedding)):
nn.init.normal_(m.weight, mean=0.0, std=0.02)
if isinstance(m, nn.Linear) and m.bias is not None:
nn.init.zeros_(m.bias)
def forward(self, idx, targets=None, loss_mask=None):
B, T = idx.shape
x = self.drop(self.tok(idx) + self.pos(torch.arange(T, device=idx.device)))
for b in self.blocks:
x = b(x)
logits = self.head(self.ln_f(x))
if targets is None:
return logits, None
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), reduction="none")
if loss_mask is not None: # Part 5: only learn from the assistant's tokens
m = loss_mask.view(-1).float()
return logits, (loss * m).sum() / m.sum().clamp(min=1)
return logits, loss.mean()
@torch.no_grad()
def generate(self, idx, max_new_tokens: int, temperature: float = 0.8, top_k: int | None = 40, stop_id: int | None = None):
self.eval()
for _ in range(max_new_tokens):
logits, _ = self(idx[:, -self.config.block_size :])
logits = logits[:, -1, :]
if temperature == 0:
nxt = logits.argmax(-1, keepdim=True) # greedy
else:
logits = logits / temperature
if top_k:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float("inf")
nxt = torch.multinomial(F.softmax(logits, dim=-1), 1)
idx = torch.cat([idx, nxt], dim=1)
if stop_id is not None and int(nxt) == stop_id:
break
return idx
def num_params(self) -> int:
return sum(p.numel() for p in self.parameters())
def unigram_loss(ids, vocab_size: int) -> float:
"""Cross-entropy of always predicting the corpus token frequencies: the bar a model must beat."""
counts = torch.bincount(torch.as_tensor(ids), minlength=vocab_size).float() + 1
p = counts / counts.sum()
return float(-(p * p.log()).sum())
@torch.no_grad()
def estimate_loss(model, data, block: int, batch: int, iters: int, gen: torch.Generator) -> float:
model.eval()
losses = []
for _ in range(iters):
ix = torch.randint(0, len(data) - block - 1, (batch,), generator=gen)
x = torch.stack([data[i : i + block] for i in ix])
y = torch.stack([data[i + 1 : i + block + 1] for i in ix])
losses.append(model(x, y)[1].item())
model.train()
return sum(losses) / len(losses)
def train_lm(model, train_ids, val_ids, steps: int = 2000, batch: int = 32, lr: float = 3e-3, warmup: int = 100,
eval_every: int = 250, seed: int = 0, log=print) -> list:
"""AdamW, linear warmup then cosine decay to 10%, gradient clipping. Restores the weights with the best
validation loss (early stopping by checkpoint). Returns the (step, train, val) history."""
gen = torch.Generator().manual_seed(seed)
tr, va = torch.as_tensor(train_ids), torch.as_tensor(val_ids)
block = model.config.block_size
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.1, betas=(0.9, 0.95))
def lr_at(s):
if s < warmup:
return lr * (s + 1) / warmup
p = (s - warmup) / max(1, steps - warmup)
return lr * (0.1 + 0.9 * 0.5 * (1 + math.cos(math.pi * p)))
history, best_val, best_state = [], float("inf"), None
model.train()
for s in range(steps):
for g in opt.param_groups:
g["lr"] = lr_at(s)
ix = torch.randint(0, len(tr) - block - 1, (batch,), generator=gen)
x = torch.stack([tr[i : i + block] for i in ix])
y = torch.stack([tr[i + 1 : i + block + 1] for i in ix])
_, loss = model(x, y)
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
if s % eval_every == 0 or s == steps - 1:
v = estimate_loss(model, va, block, batch, 10, gen)
history.append((s, loss.item(), v))
mark = ""
if v < best_val: # keep the weights that generalize best, not the last ones
best_val, best_state, mark = v, {k: x.detach().clone() for k, x in model.state_dict().items()}, " (best so far)"
log(f"step {s:5d} train {loss.item():.3f} val {v:.3f} lr {lr_at(s):.2e}{mark}")
if best_state is not None:
model.load_state_dict(best_state)
return history