Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Tensor data loaders: parsing Markdown lessons into structured text sections, QA pairs, and integer batches.
"""Part 1 · Data as tensors.
Load the lesson corpus, turn markdown into clean text, split it into sections and Q&A pairs,
and turn text into arrays of integers that the rest of Atlas trains on.
"""
from __future__ import annotations
import pathlib
import re
from dataclasses import dataclass, field
import numpy as np
DATA = pathlib.Path(__file__).resolve().parents[1] / "data" / "lessons"
@dataclass
class Lesson:
slug: str
title: str
markdown: str
sections: list = field(default_factory=list) # (heading path, text)
qa: list = field(default_factory=list) # (question, answer)
def clean_markdown(md: str) -> str:
"""Markdown to plain text: drop code fences' language tags, images, quiz blocks and HTML, keep the words."""
md = re.sub(r"```quiz\n.*?```", "", md, flags=re.S) # interactive quizzes are not prose
md = re.sub(r"```[a-z]*( no-run)?\n", "", md).replace("```", "")
md = re.sub(r"^> \*\*\[IMAGE[^\n]*$", "", md, flags=re.M)
md = re.sub(r"<[^>]+>", "", md)
md = re.sub(r"\*\*(.+?)\*\*", r"\1", md)
md = re.sub(r"`([^`]*)`", r"\1", md)
md = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", md)
md = re.sub(r"\n{3,}", "\n\n", md)
return md.strip()
def split_sections(md: str, title: str) -> list:
"""Split on ## / ### headings, keeping the heading path (useful context for retrieval later)."""
out, path, buf = [], [title], []
for line in md.splitlines():
m = re.match(r"^(#{2,3}) (.+)$", line)
if m:
if buf and "".join(buf).strip():
out.append((" > ".join(path), clean_markdown("\n".join(buf))))
level = len(m.group(1))
path = path[: level - 1] + [m.group(2).strip()]
buf = []
else:
buf.append(line)
if buf and "".join(buf).strip():
out.append((" > ".join(path), clean_markdown("\n".join(buf))))
return [(h, t) for h, t in out if len(t) > 40]
def extract_qa(md: str) -> list:
"""The 'Check yourself' questions paired with their answers (numbered lists in the same order)."""
m = re.search(r"## Check yourself\n(.*?)<details><summary>Answers</summary>\n(.*?)</details>", md, re.S)
if not m:
return []
def items(block: str) -> list:
parts = re.split(r"^\d+\.\s+", block.strip(), flags=re.M)
return [clean_markdown(p).replace("\n", " ").strip() for p in parts if p.strip()]
qs, ans = items(m.group(1)), items(m.group(2))
return [(q, a) for q, a in zip(qs, ans) if q and a]
def load_lessons(path: pathlib.Path = DATA) -> list:
lessons = []
for f in sorted(path.glob("*.md")):
md = f.read_text()
title = md.splitlines()[0].lstrip("# ").strip()
lessons.append(Lesson(f.stem, title, md, split_sections(md, title), extract_qa(md)))
return lessons
def corpus_text(lessons: list) -> str:
"""All lessons as one clean string: the pretraining text for TinyGPT (Part 4)."""
return "\n\n".join(clean_markdown(l.markdown) for l in lessons)
def train_val_split(ids: np.ndarray, val_fraction: float = 0.1) -> tuple:
"""Split a token stream by position (the end is validation), never randomly: random splits leak context."""
n = int(len(ids) * (1 - val_fraction))
return ids[:n], ids[n:]
def get_batch(ids: np.ndarray, block: int, batch: int, rng: np.random.Generator) -> tuple:
"""Random windows of `block` tokens; y is x shifted by one (next-token targets). Vectorized, no Python loop over tokens."""
starts = rng.integers(0, len(ids) - block - 1, size=batch)
offsets = np.arange(block)
x = ids[starts[:, None] + offsets] # (batch, block) by broadcasting
y = ids[starts[:, None] + offsets + 1]
return x, y