Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Runnable checkpoint: simulates concurrent user traffic and measures Time-To-First-Token (TTFT) and throughput.
"""Part 9 · Serve it fast: KV-cache decoding, batching and cost per token for TinyGPT."""
import time
import common
import numpy as np
import torch
from atlas.gpt import GPTConfig, TinyGPT
from atlas.serving import generate_cached, generate_naive, request_trace, simulate, time_decode_step
common.banner("Part 9 · Serve it fast")
torch.manual_seed(0)
ckpt = common.RUNS / "tinygpt.pt"
if ckpt.exists():
state = torch.load(ckpt, map_location="cpu")
model = TinyGPT(GPTConfig(**state["config"])); model.load_state_dict(state["state"])
print("using the TinyGPT trained in Part 4")
else:
model = TinyGPT(GPTConfig(vocab_size=1024)) # same shape as Part 4; speed doesn't depend on the weights
print("Part 4 checkpoint not found: using an untrained TinyGPT of the same size (speed is identical)")
model.eval()
c = model.config
print(f"{model.num_params():,} parameters · {c.n_layer} layers · d_model {c.d_model} · context {c.block_size}")
# 1. The KV cache: same tokens, far less work per step
prompt = torch.randint(0, c.vocab_size, (1, 16))
same = torch.equal(generate_cached(model, prompt, 100), generate_naive(model, prompt, 100))
for name, fn in [("no cache (recompute everything)", generate_naive), ("KV cache", generate_cached)]:
t = time.perf_counter(); out = fn(model, prompt, 100); dt = time.perf_counter() - t
print(f"{name:<34} {out.size(1) - 16} tokens in {dt * 1000:6.0f} ms -> {(out.size(1) - 16) / dt:6.0f} tokens/s")
print(f"identical outputs: {same}")
kv_bytes = 2 * c.n_layer * c.d_model * 4
print(f"KV cache costs {kv_bytes:,} bytes per token ({kv_bytes * c.block_size / 1024:.0f} KB for a full {c.block_size}-token context, fp32)")
# 2. Batching: one decode step serves the whole batch
print(f"\n{'batch':>6}{'ms/step':>9}{'tokens/s':>10}")
measured = {}
for b in [1, 4, 16, 64]:
measured[b] = time_decode_step(model, b)
print(f"{b:>6}{measured[b] * 1000:>9.2f}{b / measured[b]:>10.0f}")
bs = np.array(list(measured)); ts = np.array(list(measured.values()))
slope, intercept = np.polyfit(bs, ts, 1)
step_time = lambda b: intercept + slope * b
print(f"fitted step time: {intercept * 1000:.2f} ms + {slope * 1000:.3f} ms per sequence in the batch")
# 3. Static vs continuous batching on a saturating request trace (decode only; outputs of 10-100 tokens)
trace = request_trace(n=600, rate_per_s=400.0) # a busy hour: more requests than one-at-a-time can serve
print(f"\n{'scheduler':<26}{'tokens/s':>10}{'p50 latency':>13}{'p95 latency':>13}")
for name, kw in [("one request at a time", dict(max_batch=1, continuous=False)),
("static batches of 16", dict(max_batch=16, continuous=False)),
("continuous, 16 slots", dict(max_batch=16, continuous=True))]:
r = simulate(trace, step_time, **kw)
print(f"{name:<26}{r['throughput']:>10.0f}{r['p50']:>12.2f}s{r['p95']:>12.2f}s")
# 4. Cost per token
price_per_hour = 0.10 # a small CPU VM; change to your hardware
for name, b in [("batch 1", 1), ("batch 16", 16), ("batch 64", 64)]:
tps = b / step_time(b)
print(f"cost at {name:<9} ${price_per_hour / (tps * 3600) * 1e6:.3f} per million tokens ({tps:,.0f} tokens/s)")
print("\nReading it: the cache makes each decode step cost one token's work instead of the whole sequence's, with")
print("identical output. Batching shares the fixed per-step cost (reading the weights) across sequences, and")
print("continuous batching keeps slots busy instead of waiting for each batch's longest request, which helps both")
print("throughput and latency. Cost per token is just the price of the machine divided by the tokens it produces.")
print("\nCheckpoint: Atlas generates with a KV cache, serves batched traffic and knows its cost per token.")