Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
TF-IDF featurizer and multi-class softmax regression classifier implemented from first principles in NumPy.
"""Part 2 · Learning from data.
A baseline you can trust before any neural network: TF-IDF features, softmax regression trained with
gradient descent in NumPy, and an evaluation that holds out whole lessons (no leakage from a lesson's
own vocabulary) and always reports the majority-class baseline next to the model.
"""
from __future__ import annotations
import re
from collections import Counter
import numpy as np
# Topic of each free lesson. The task: given a section of text, which topic is it about?
TOPICS = ["python-language", "python-performance", "systems"]
TOPIC_OF = {
**{s: "python-language" for s in [
"py-values-and-names", "py-control-flow", "py-functions", "py-lists-tuples", "py-dicts-sets", "py-strings-unicode",
"py-iteration", "py-decorators-context", "py-classes-types", "py-testing-packaging"]},
**{s: "python-performance" for s in [
"python-native-internals", "py-gil-concurrency", "py-profiling", "py-async-concurrency", "py-numpy-pandas", "cpp-systems-memory"]},
**{s: "systems" for s in [
"ai-data-structures-algorithms", "operating-systems-memory-paging", "high-throughput-networking", "databases-and-model-registry"]},
}
# Held-out lessons: the model never sees any of their text during training.
TEST_LESSONS = {"py-classes-types", "py-dicts-sets", "py-profiling", "py-async-concurrency", "databases-and-model-registry"}
WORD = re.compile(r"[a-z_][a-z0-9_]+")
def tokenize(text: str) -> list:
return WORD.findall(text.lower())
class TfidfVectorizer:
"""Bag of words weighted by inverse document frequency; rows L2-normalized."""
def __init__(self, max_features: int = 3000, min_df: int = 2):
self.max_features, self.min_df = max_features, min_df
def fit(self, docs: list) -> "TfidfVectorizer":
df = Counter(w for d in docs for w in set(tokenize(d)))
vocab = [w for w, c in df.most_common() if c >= self.min_df][: self.max_features]
self.index = {w: i for i, w in enumerate(vocab)}
n = len(docs)
self.idf = np.array([np.log((1 + n) / (1 + df[w])) + 1 for w in vocab])
return self
def transform(self, docs: list) -> np.ndarray:
X = np.zeros((len(docs), len(self.index)))
for r, d in enumerate(docs):
for w, c in Counter(tokenize(d)).items():
j = self.index.get(w)
if j is not None:
X[r, j] = c
X = np.log1p(X) * self.idf
return X / np.maximum(np.linalg.norm(X, axis=1, keepdims=True), 1e-12)
def softmax(z: np.ndarray) -> np.ndarray:
z = z - z.max(axis=1, keepdims=True) # stable: subtract the max before exp
e = np.exp(z)
return e / e.sum(axis=1, keepdims=True)
class SoftmaxRegression:
"""Multinomial logistic regression, full-batch gradient descent with L2 regularization."""
def __init__(self, lr: float = 2.0, l2: float = 1e-3, steps: int = 400):
self.lr, self.l2, self.steps = lr, l2, steps
def fit(self, X: np.ndarray, y: np.ndarray, k: int) -> "SoftmaxRegression":
n, d = X.shape
self.W, self.b = np.zeros((d, k)), np.zeros(k)
Y = np.eye(k)[y]
self.losses = []
for _ in range(self.steps):
P = softmax(X @ self.W + self.b)
self.losses.append(-np.mean(np.log(P[np.arange(n), y] + 1e-12)))
G = (P - Y) / n # gradient of mean cross-entropy w.r.t. the logits
self.W -= self.lr * (X.T @ G + self.l2 * self.W)
self.b -= self.lr * G.sum(axis=0)
return self
def predict(self, X: np.ndarray) -> np.ndarray:
return (X @ self.W + self.b).argmax(axis=1)
def macro_f1(y: np.ndarray, pred: np.ndarray, k: int) -> float:
f1s = []
for c in range(k):
tp = np.sum((pred == c) & (y == c))
fp = np.sum((pred == c) & (y != c))
fn = np.sum((pred != c) & (y == c))
p = tp / (tp + fp) if tp + fp else 0.0
r = tp / (tp + fn) if tp + fn else 0.0
f1s.append(2 * p * r / (p + r) if p + r else 0.0)
return float(np.mean(f1s))
def confusion(y: np.ndarray, pred: np.ndarray, k: int) -> np.ndarray:
m = np.zeros((k, k), dtype=int)
np.add.at(m, (y, pred), 1)
return m
def dataset(lessons: list) -> tuple:
"""(texts, labels, lesson slugs) for every section of a lesson with a known topic."""
texts, labels, slugs = [], [], []
for l in lessons:
if l.slug not in TOPIC_OF:
continue
for _, text in l.sections:
texts.append(text)
labels.append(TOPICS.index(TOPIC_OF[l.slug]))
slugs.append(l.slug)
return texts, np.array(labels), slugs
def choose_l2(texts: list, y: np.ndarray, slugs: list, grid=(1e-3, 1e-2, 3e-2, 1e-1)) -> tuple:
"""Pick L2 strength by grouped cross-validation over *training lessons only*: each fold holds out whole
lessons, mirroring the test setup. The test set is never used for tuning."""
lessons = sorted(set(slugs))
folds = [lessons[i::4] for i in range(4)]
scores = {}
for l2 in grid:
f1s = []
for held in folds:
va = np.array([s in held for s in slugs])
vec = TfidfVectorizer().fit([t for t, v in zip(texts, va) if not v])
X = vec.transform(texts)
m = SoftmaxRegression(l2=l2).fit(X[~va], y[~va], len(TOPICS))
f1s.append(macro_f1(y[va], m.predict(X[va]), len(TOPICS)))
scores[l2] = float(np.mean(f1s))
return max(scores, key=scores.get), scores
def run(lessons: list) -> dict:
texts, y, slugs = dataset(lessons)
test = np.array([s in TEST_LESSONS for s in slugs])
tr = [i for i, te in enumerate(test) if not te]
l2, cv = choose_l2([texts[i] for i in tr], y[tr], [slugs[i] for i in tr])
vec = TfidfVectorizer().fit([texts[i] for i in tr]) # fit on training text only
X = vec.transform(texts)
model = SoftmaxRegression(l2=l2).fit(X[~test], y[~test], len(TOPICS))
pred = model.predict(X[test])
majority = np.bincount(y[~test]).argmax()
return {
"train_sections": int((~test).sum()),
"test_sections": int(test.sum()),
"accuracy": float(np.mean(pred == y[test])),
"macro_f1": macro_f1(y[test], pred, len(TOPICS)),
"majority_accuracy": float(np.mean(y[test] == majority)),
"majority_macro_f1": macro_f1(y[test], np.full(test.sum(), majority), len(TOPICS)),
"confusion": confusion(y[test], pred, len(TOPICS)),
"loss_start": model.losses[0],
"loss_end": model.losses[-1],
"l2": l2,
"cv_macro_f1": cv,
"vectorizer": vec,
"model": model,
}