Loading codebase...
Complete Source Code
Every single line of code behind Build Atlas, organized across all 9 parts. From tensor math, autograd, and byte-level BPE to TinyGPT training, LoRA chat tuning, hybrid RAG, autonomous agents, and continuous-batch serving.
Low-Rank Adaptation (LoRA) layer implementation: freezing base weights and training low-rank A/B matrices.
"""Part 5 · Make it helpful (2/2): LoRA, low-rank adaptation.
Freeze every pretrained weight W. Next to chosen linear layers, add a small trainable detour B·A of rank r,
scaled by alpha/r. B starts at zero, so training begins exactly at the base model; only A and B learn.
"""
from __future__ import annotations
import math
import torch
from torch import nn
class LoRALinear(nn.Module):
def __init__(self, base: nn.Linear, r: int = 8, alpha: float = 16.0, dropout: float = 0.0):
super().__init__()
self.base = base
for p in self.base.parameters():
p.requires_grad_(False)
self.A = nn.Parameter(torch.empty(r, base.in_features))
self.B = nn.Parameter(torch.zeros(base.out_features, r)) # zero: no change at the start
nn.init.kaiming_uniform_(self.A, a=math.sqrt(5))
self.scale = alpha / r
self.drop = nn.Dropout(dropout)
def forward(self, x):
return self.base(x) + (self.drop(x) @ self.A.T @ self.B.T) * self.scale
@torch.no_grad()
def merged(self) -> nn.Linear:
"""Fold B·A into W for serving: one matrix, zero extra latency."""
lin = nn.Linear(self.base.in_features, self.base.out_features, bias=self.base.bias is not None)
lin.weight.copy_(self.base.weight + self.scale * self.B @ self.A)
if self.base.bias is not None:
lin.bias.copy_(self.base.bias)
return lin
def apply_lora(model: nn.Module, targets=("qkv", "proj"), r: int = 8, alpha: float = 16.0) -> nn.Module:
"""Freeze the whole model, then wrap the named linear layers with LoRA."""
for p in model.parameters():
p.requires_grad_(False)
for name, module in list(model.named_modules()):
for child_name, child in list(module.named_children()):
if child_name in targets and isinstance(child, nn.Linear):
setattr(module, child_name, LoRALinear(child, r=r, alpha=alpha))
return model
def trainable_fraction(model: nn.Module) -> tuple:
total = sum(p.numel() for p in model.parameters())
train = sum(p.numel() for p in model.parameters() if p.requires_grad)
return train, total
def merge_lora(model: nn.Module) -> nn.Module:
"""Replace every LoRALinear with a plain Linear holding W + scale * B @ A (for saving and serving)."""
for name, module in list(model.named_modules()):
for child_name, child in list(module.named_children()):
if isinstance(child, LoRALinear):
setattr(module, child_name, child.merged())
for p in model.parameters():
p.requires_grad_(True)
return model