Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Reverse-mode scalar automatic differentiation engine with topological DAG sort and chain-rule propagation.
"""Part 3 · Neural networks from scratch (1/2): a tiny scalar autograd engine.
Every Value remembers how it was made. backward() walks the graph in reverse topological order and applies
the chain rule: each node adds (local derivative × upstream gradient) into its inputs' .grad.
"""
from __future__ import annotations
import math
import random
class Value:
__slots__ = ("data", "grad", "_backward", "_prev", "op")
def __init__(self, data: float, _prev=(), op: str = ""):
self.data, self.grad = float(data), 0.0
self._backward = lambda: None
self._prev, self.op = tuple(_prev), op
def __repr__(self):
return f"Value(data={self.data:.4f}, grad={self.grad:.4f})"
@staticmethod
def wrap(x) -> "Value":
return x if isinstance(x, Value) else Value(x)
def __add__(self, other):
other = Value.wrap(other)
out = Value(self.data + other.data, (self, other), "+")
def _backward():
self.grad += out.grad # d(a+b)/da = 1
other.grad += out.grad
out._backward = _backward
return out
def __mul__(self, other):
other = Value.wrap(other)
out = Value(self.data * other.data, (self, other), "*")
def _backward():
self.grad += other.data * out.grad # d(ab)/da = b
other.grad += self.data * out.grad
out._backward = _backward
return out
def __pow__(self, k: float):
out = Value(self.data**k, (self,), f"**{k}")
def _backward():
self.grad += k * self.data ** (k - 1) * out.grad
out._backward = _backward
return out
def tanh(self):
t = math.tanh(self.data)
out = Value(t, (self,), "tanh")
def _backward():
self.grad += (1 - t * t) * out.grad
out._backward = _backward
return out
def exp(self):
e = math.exp(self.data)
out = Value(e, (self,), "exp")
def _backward():
self.grad += e * out.grad
out._backward = _backward
return out
def log(self):
out = Value(math.log(self.data), (self,), "log")
def _backward():
self.grad += out.grad / self.data
out._backward = _backward
return out
def __neg__(self):
return self * -1
def __sub__(self, other):
return self + (-Value.wrap(other))
def __truediv__(self, other):
return self * Value.wrap(other) ** -1
__radd__ = __add__
__rmul__ = __mul__
def __rsub__(self, other):
return Value.wrap(other) - self
def backward(self):
order, seen = [], set()
def build(v):
if id(v) not in seen:
seen.add(id(v))
for p in v._prev:
build(p)
order.append(v)
build(self)
self.grad = 1.0
for v in reversed(order):
v._backward()
class Neuron:
def __init__(self, n_in: int, rng: random.Random):
self.w = [Value(rng.uniform(-1, 1) / math.sqrt(n_in)) for _ in range(n_in)]
self.b = Value(0.0)
def __call__(self, x):
return sum((wi * xi for wi, xi in zip(self.w, x)), self.b).tanh()
def parameters(self):
return self.w + [self.b]
class MLP:
"""Layers of tanh neurons; the last layer is linear (logits)."""
def __init__(self, sizes: list, seed: int = 0):
rng = random.Random(seed)
self.layers = [[Neuron(a, rng) for _ in range(b)] for a, b in zip(sizes, sizes[1:])]
def __call__(self, x):
for i, layer in enumerate(self.layers):
if i == len(self.layers) - 1:
x = [sum((wi * xi for wi, xi in zip(n.w, x)), n.b) for n in layer] # linear output
else:
x = [n(x) for n in layer]
return x
def parameters(self):
return [p for layer in self.layers for n in layer for p in n.parameters()]
def cross_entropy(logits: list, target: int) -> Value:
m = max(l.data for l in logits)
exps = [(l - m).exp() for l in logits] # subtract the max for stability
return -((exps[target] / sum(exps[1:], exps[0])).log())
def numerical_grad(f, xs: list, eps: float = 1e-6) -> list:
"""Finite-difference gradient: the ground truth our backward() must match."""
grads = []
for i in range(len(xs)):
up = xs[:i] + [xs[i] + eps] + xs[i + 1 :]
dn = xs[:i] + [xs[i] - eps] + xs[i + 1 :]
grads.append((f(up) - f(dn)) / (2 * eps))
return grads