Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Runnable checkpoint: applies LoRA to TinyGPT, fine-tunes on QA instructions, and benchmarks loss vs full fine-tuning.
"""Part 5 · Make it helpful: fine-tune TinyGPT to answer questions, fully and with LoRA. Needs Part 4."""
import copy
import common
import torch
from load import load_tinygpt
from atlas.data import load_lessons
from atlas.lora import apply_lora, merge_lora, trainable_fraction
from atlas.sft import answer_loss, chat, split_qa, train_sft
common.banner("Part 5 · Make it helpful")
torch.manual_seed(0)
base, tok = load_tinygpt()
qa = [p for l in load_lessons() for p in l.qa]
train_qa, val_qa = split_qa(qa)
print(f"{len(qa)} Q&A pairs from 'Check yourself' · fine-tune on {len(train_qa)} · validate on {len(val_qa)}")
full = copy.deepcopy(base)
print("\nfull fine-tuning (every weight trains):")
train_sft(full, tok, train_qa, steps=300, lr=5e-4)
lora = apply_lora(copy.deepcopy(base), r=8)
n_train, n_total = trainable_fraction(lora)
print(f"\nLoRA r=8 on attention (trains {n_train:,} of {n_total:,} parameters, {n_train / n_total:.1%}):")
train_sft(lora, tok, train_qa, steps=300, lr=2e-3)
print(f"\nloss on the answers (lower = right answers more likely): validation ({len(val_qa)} unseen) · training")
for name, m in [("base model (no fine-tuning)", base), ("full fine-tuning", full), ("LoRA", lora)]:
print(f" {name:<30}{answer_loss(m, tok, val_qa):.3f} train-set {answer_loss(m, tok, train_qa):.3f}")
merged = merge_lora(copy.deepcopy(lora))
assert abs(answer_loss(merged, tok, val_qa) - answer_loss(lora, tok, val_qa)) < 1e-4 # merging changes nothing
q, a = val_qa[0]
print(f"\nunseen question: {q}\nreference: {a}\nLoRA model says: {chat(merged, tok, q)!r}")
print("\nReading it: full fine-tuning memorizes the 62 training answers (loss near 0) and gets *worse* on unseen")
print("questions than the base model: classic overfitting. LoRA updates 2.5% of the weights, can't memorize as")
print("much, and improves unseen answers. On tiny data, a small update is a regularizer. Still, a 1M-parameter")
print("model knows little, which is why Part 7 gives Atlas the documents instead of hoping it memorized them.")
torch.save({"state": merged.state_dict(), "config": merged.config.__dict__}, common.RUNS / "atlas-chat.pt")
print(f"\nCheckpoint: Atlas's chat model (LoRA, merged into the weights) saved to {common.RUNS / 'atlas-chat.pt'}")