Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Hybrid retrieval engine combining BM25 keyword matching and dense LSA vectors via Reciprocal Rank Fusion (RRF).
"""Part 7 · Ground it in your documents: retrieval-augmented generation.
Chunk the lessons by structure (keeping each chunk's heading path), index them two ways (BM25 for exact
terms, LSA vectors for meaning), fuse the rankings, and hand the top chunks to the model with citations.
Retrieval is measured on its own (recall@k, MRR) before anyone blames the generator.
"""
from __future__ import annotations
import math
import re
from collections import Counter
from dataclasses import dataclass
import numpy as np
from atlas.prompts import Doc, build_prompt
TOKEN = re.compile(r"[a-z0-9_]+")
STOP = set("the a an of to in is are and or for on with it this that be as by at from you your".split())
def terms(text: str) -> list:
return [t for t in TOKEN.findall(text.lower()) if t not in STOP]
@dataclass
class Chunk:
lesson: str # slug
heading: str # "Lesson title > section > subsection"
text: str
@property
def indexed(self) -> str:
return f"{self.heading}\n{self.text}" # the heading travels with the text (contextual chunking)
def chunk_lessons(lessons: list, max_words: int = 180) -> list:
"""One chunk per section; long sections are split on paragraph boundaries, never mid-sentence."""
chunks = []
for l in lessons:
for heading, text in l.sections:
if heading.endswith("Check yourself") or "Answers" in heading:
continue # don't let retrieval find the answer key itself
paras, buf, n = text.split("\n\n"), [], 0
for p in paras:
w = len(p.split())
if buf and n + w > max_words:
chunks.append(Chunk(l.slug, heading, "\n\n".join(buf)))
buf, n = [], 0
buf.append(p)
n += w
if buf:
chunks.append(Chunk(l.slug, heading, "\n\n".join(buf)))
return chunks
class BM25:
def __init__(self, docs: list, k1: float = 1.5, b: float = 0.75):
self.k1, self.b = k1, b
self.tfs = [Counter(terms(d)) for d in docs]
self.lens = np.array([sum(tf.values()) for tf in self.tfs], dtype=float)
self.avg = self.lens.mean()
df = Counter(t for tf in self.tfs for t in tf)
n = len(docs)
self.idf = {t: math.log(1 + (n - c + 0.5) / (c + 0.5)) for t, c in df.items()}
def scores(self, query: str) -> np.ndarray:
q = terms(query)
s = np.zeros(len(self.tfs))
for i, tf in enumerate(self.tfs):
norm = self.k1 * (1 - self.b + self.b * self.lens[i] / self.avg)
s[i] = sum(self.idf.get(t, 0.0) * tf[t] * (self.k1 + 1) / (tf[t] + norm) for t in q if t in tf)
return s
class LSA:
"""Dense vectors without a neural network: TF-IDF, then a truncated SVD to k latent dimensions."""
def __init__(self, docs: list, k: int = 96, min_df: int = 2):
df = Counter(t for d in docs for t in set(terms(d)))
self.vocab = {t: i for i, t in enumerate(t for t, c in df.items() if c >= min_df)}
n = len(docs)
self.idf = np.zeros(len(self.vocab))
for t, i in self.vocab.items():
self.idf[i] = math.log((1 + n) / (1 + df[t])) + 1
X = self._tfidf(docs)
U, S, Vt = np.linalg.svd(X, full_matrices=False)
k = min(k, len(S))
self.proj = Vt[:k].T / S[:k] # maps a tf-idf row into the latent space
self.vecs = self._norm(X @ self.proj)
def _tfidf(self, docs: list) -> np.ndarray:
X = np.zeros((len(docs), len(self.vocab)))
for r, d in enumerate(docs):
for t, c in Counter(terms(d)).items():
j = self.vocab.get(t)
if j is not None:
X[r, j] = 1 + math.log(c)
X *= self.idf
return X / np.maximum(np.linalg.norm(X, axis=1, keepdims=True), 1e-12)
@staticmethod
def _norm(V: np.ndarray) -> np.ndarray:
return V / np.maximum(np.linalg.norm(V, axis=1, keepdims=True), 1e-12)
def scores(self, query: str) -> np.ndarray:
q = self._norm(self._tfidf([query]) @ self.proj)
return self.vecs @ q[0] # cosine similarity
def rrf(rankings: list, k: int = 60) -> np.ndarray:
"""Reciprocal rank fusion: robust to scores on different scales; only ranks matter."""
fused = np.zeros(len(rankings[0]))
for scores in rankings:
order = np.argsort(-scores)
ranks = np.empty_like(order)
ranks[order] = np.arange(len(order))
fused += 1.0 / (k + ranks + 1)
return fused
class Retriever:
def __init__(self, chunks: list):
self.chunks = chunks
texts = [c.indexed for c in chunks]
self.bm25, self.lsa = BM25(texts), LSA(texts)
def search(self, query: str, k: int = 5, mode: str = "hybrid") -> list:
if mode == "bm25":
s = self.bm25.scores(query)
elif mode == "dense":
s = self.lsa.scores(query)
else:
s = rrf([self.bm25.scores(query), self.lsa.scores(query)])
return [self.chunks[i] for i in np.argsort(-s)[:k]]
def retrieval_metrics(retriever: Retriever, cases: list, k: int = 5, mode: str = "hybrid") -> dict:
"""recall@k: the gold lesson appears in the top k; MRR: 1 / rank of the first gold chunk (0 if absent)."""
hits, rr = 0, 0.0
for c in cases:
got = retriever.search(c.question, k=k, mode=mode)
ranks = [i for i, ch in enumerate(got) if ch.lesson == c.source]
if ranks:
hits += 1
rr += 1.0 / (ranks[0] + 1)
return {"mode": mode, f"recall@{k}": hits / len(cases), "mrr": rr / len(cases)}
def answer(question: str, retriever: Retriever, llm, k: int = 3) -> tuple:
chunks = retriever.search(question, k=k)
prompt = build_prompt(question, [Doc(f"{c.lesson} > {c.heading}", c.text) for c in chunks])
return llm.complete(prompt), chunks