Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Reusable optimization loops: mini-batch SGD, momentum, weight decay, and gradient clipping.
"""Part 3 · Neural networks from scratch (2/2): the same idea in PyTorch, with a careful training loop.
Our autograd engine shows *what* backprop does. PyTorch does the same on tensors, fast. This loop is the
template every later part reuses: seed everything, train in batches, evaluate without gradients, keep the
best weights by validation score (early stopping), and never touch the test set while choosing.
"""
from __future__ import annotations
import copy
import numpy as np
import torch
from torch import nn
def seed_everything(seed: int = 0) -> None:
np.random.seed(seed)
torch.manual_seed(seed)
class TopicMLP(nn.Module):
def __init__(self, d_in: int, d_hidden: int, k: int, dropout: float = 0.3):
super().__init__()
self.net = nn.Sequential(nn.Linear(d_in, d_hidden), nn.ReLU(), nn.Dropout(dropout), nn.Linear(d_hidden, k))
def forward(self, x):
return self.net(x)
@torch.no_grad()
def evaluate(model: nn.Module, X: torch.Tensor, y: torch.Tensor) -> tuple:
model.eval() # dropout off
logits = model(X)
loss = nn.functional.cross_entropy(logits, y).item()
pred = logits.argmax(1)
return loss, pred.cpu().numpy()
def fit(model: nn.Module, Xtr, ytr, Xva, yva, score_fn, epochs: int = 200, lr: float = 3e-3, wd: float = 1e-2,
batch: int = 32, patience: int = 25, seed: int = 0) -> dict:
"""Mini-batch AdamW with early stopping on a validation score (higher is better)."""
gen = torch.Generator().manual_seed(seed)
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)
best, best_state, since, history = -1.0, None, 0, []
for epoch in range(epochs):
model.train()
perm = torch.randperm(len(Xtr), generator=gen)
for i in range(0, len(Xtr), batch):
idx = perm[i : i + batch]
loss = nn.functional.cross_entropy(model(Xtr[idx]), ytr[idx])
opt.zero_grad(set_to_none=True) # forget last step's gradients
loss.backward()
opt.step()
va_loss, va_pred = evaluate(model, Xva, yva)
score = score_fn(yva.numpy(), va_pred)
history.append((epoch, float(loss.item()), va_loss, score))
if score > best:
best, best_state, since = score, copy.deepcopy(model.state_dict()), 0
else:
since += 1
if since >= patience:
break
model.load_state_dict(best_state)
return {"best_val_score": best, "epochs_run": len(history), "history": history}