768
La
learning-ai-lua-code-model / files
learning-ai-lua-code-model / src / lua_code_model / train_gpt.py
15.5 KB
1"""Train a tiny GPT-style Transformer language model with PyTorch. 2 3This is the next step after the fixed-context MLP. 4 5The MLP saw a fixed window and flattened it: 6 7 previous 32 characters -> one long vector -> next character 8 9This GPT-style model keeps the sequence structure. Each position can use 10causal self-attention to decide which earlier positions are useful: 11 12 previous characters -> attention over earlier characters -> next character 13 14It is still tiny and educational, not a useful coding assistant. 15 16Run from the project root after preparing the corpus: 17 18 PYTHONPATH=src uv run python -m lua_code_model.train_gpt 19""" 20 21from __future__ import annotations 22 23import json 24from pathlib import Path 25 26import torch 27import torch.nn as nn 28import torch.nn.functional as F 29 30 31PROJECT_ROOT = Path(__file__).resolve().parents[2] 32PROCESSED_DIR = PROJECT_ROOT / "data" / "processed" 33CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints" 34 35TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt" 36VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt" 37VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json" 38CHECKPOINT_PATH = CHECKPOINT_DIR / "gpt_tiny.pt" 39 40# Tiny GPT hyperparameters. These are deliberately small enough to study. 41BATCH_SIZE = 16 42BLOCK_SIZE = 64 43MAX_STEPS = 2_000 44EVAL_INTERVAL = 200 45EVAL_ITERS = 20 46LEARNING_RATE = 1e-3 47RANDOM_SEED = 1337 48 49N_EMBD = 64 50N_HEAD = 4 51N_LAYER = 2 52DROPOUT = 0.0 53 54SAMPLE_START_TEXT = "local function" 55SAMPLE_NEW_TOKENS = 500 56SAMPLE_TEMPERATURE = 0.8 57 58 59class CausalSelfAttentionHead(nn.Module): 60 """One self-attention head. 61 62 Self-attention lets each token position gather information from other token 63 positions in the same sequence. 64 65 This is a *causal* head because it is masked so a position can only look at 66 itself and earlier positions. That keeps the model honest for next-token 67 prediction: it cannot peek at future answers. 68 """ 69 70 def __init__(self, n_embd: int, head_size: int, block_size: int, dropout: float) -> None: 71 super().__init__() 72 self.key = nn.Linear(n_embd, head_size, bias=False) 73 self.query = nn.Linear(n_embd, head_size, bias=False) 74 self.value = nn.Linear(n_embd, head_size, bias=False) 75 self.dropout = nn.Dropout(dropout) 76 77 # Lower-triangular mask. For a sequence of length T, tril[:T, :T] 78 # allows attention to current and previous positions, but blocks future 79 # positions. 80 self.register_buffer("tril", torch.tril(torch.ones(block_size, block_size))) 81 82 def forward(self, x: torch.Tensor) -> torch.Tensor: 83 batch_size, sequence_length, channels = x.shape 84 85 keys = self.key(x) # batch_size x sequence_length x head_size 86 queries = self.query(x) # batch_size x sequence_length x head_size 87 88 # Attention scores answer: how relevant is each key to each query? 89 # The scaling keeps values numerically stable as head_size grows. 90 attention_scores = queries @ keys.transpose(-2, -1) * (keys.shape[-1] ** -0.5) 91 92 # Hide future positions by setting their scores to -infinity. After 93 # softmax, those positions receive probability 0. 94 attention_scores = attention_scores.masked_fill( 95 self.tril[:sequence_length, :sequence_length] == 0, 96 float("-inf"), 97 ) 98 99 attention_weights = F.softmax(attention_scores, dim=-1) 100 attention_weights = self.dropout(attention_weights) 101 102 values = self.value(x) 103 output = attention_weights @ values 104 return output 105 106 107class MultiHeadCausalSelfAttention(nn.Module): 108 """Run several attention heads in parallel and combine their outputs. 109 110 Each head can learn a different kind of relationship. For example, one head 111 might focus on nearby syntax while another focuses on earlier names or 112 indentation-like patterns. 113 """ 114 115 def __init__(self, n_embd: int, n_head: int, block_size: int, dropout: float) -> None: 116 super().__init__() 117 if n_embd % n_head != 0: 118 raise ValueError(f"n_embd={n_embd} must be divisible by n_head={n_head}") 119 120 head_size = n_embd // n_head 121 self.heads = nn.ModuleList( 122 [CausalSelfAttentionHead(n_embd, head_size, block_size, dropout) for _ in range(n_head)] 123 ) 124 self.projection = nn.Linear(n_embd, n_embd) 125 self.dropout = nn.Dropout(dropout) 126 127 def forward(self, x: torch.Tensor) -> torch.Tensor: 128 head_outputs = [head(x) for head in self.heads] 129 concatenated = torch.cat(head_outputs, dim=-1) 130 projected = self.projection(concatenated) 131 return self.dropout(projected) 132 133 134class FeedForward(nn.Module): 135 """A small MLP applied independently to each token position. 136 137 Attention mixes information across positions. The feed-forward network then 138 gives each position more nonlinear processing capacity. 139 """ 140 141 def __init__(self, n_embd: int, dropout: float) -> None: 142 super().__init__() 143 self.network = nn.Sequential( 144 nn.Linear(n_embd, 4 * n_embd), 145 nn.ReLU(), 146 nn.Linear(4 * n_embd, n_embd), 147 nn.Dropout(dropout), 148 ) 149 150 def forward(self, x: torch.Tensor) -> torch.Tensor: 151 return self.network(x) 152 153 154class TransformerBlock(nn.Module): 155 """One GPT-style Transformer block. 156 157 A block has two main sub-layers: 158 159 1. causal self-attention: tokens exchange information with earlier tokens 160 2. feed-forward MLP: each token's representation is processed further 161 162 Residual connections add each sub-layer's output back to its input. LayerNorm 163 keeps activations in a healthier range for training. 164 """ 165 166 def __init__(self, n_embd: int, n_head: int, block_size: int, dropout: float) -> None: 167 super().__init__() 168 self.self_attention = MultiHeadCausalSelfAttention(n_embd, n_head, block_size, dropout) 169 self.feed_forward = FeedForward(n_embd, dropout) 170 self.layer_norm_1 = nn.LayerNorm(n_embd) 171 self.layer_norm_2 = nn.LayerNorm(n_embd) 172 173 def forward(self, x: torch.Tensor) -> torch.Tensor: 174 # Pre-norm Transformer style: 175 # normalize -> sub-layer -> residual add 176 x = x + self.self_attention(self.layer_norm_1(x)) 177 x = x + self.feed_forward(self.layer_norm_2(x)) 178 return x 179 180 181class TinyGPTLanguageModel(nn.Module): 182 """A tiny character-level GPT-style language model.""" 183 184 def __init__( 185 self, 186 vocab_size: int, 187 block_size: int, 188 n_embd: int, 189 n_head: int, 190 n_layer: int, 191 dropout: float, 192 ) -> None: 193 super().__init__() 194 self.block_size = block_size 195 196 self.token_embedding_table = nn.Embedding(vocab_size, n_embd) 197 self.position_embedding_table = nn.Embedding(block_size, n_embd) 198 self.blocks = nn.Sequential( 199 *[TransformerBlock(n_embd, n_head, block_size, dropout) for _ in range(n_layer)] 200 ) 201 self.final_layer_norm = nn.LayerNorm(n_embd) 202 self.language_model_head = nn.Linear(n_embd, vocab_size) 203 204 def forward( 205 self, 206 input_ids: torch.Tensor, 207 targets: torch.Tensor | None = None, 208 ) -> tuple[torch.Tensor, torch.Tensor | None]: 209 """Return logits, and optionally loss if targets are provided. 210 211 input_ids shape: 212 213 batch_size x block_size 214 215 logits shape: 216 217 batch_size x block_size x vocab_size 218 219 Unlike the fixed-context MLP, GPT predicts the next token at every 220 position in the block, not only after the block. 221 """ 222 223 batch_size, sequence_length = input_ids.shape 224 if sequence_length > self.block_size: 225 raise ValueError(f"Cannot process sequence length {sequence_length}; block_size is {self.block_size}") 226 227 token_embeddings = self.token_embedding_table(input_ids) 228 positions = torch.arange(sequence_length) 229 position_embeddings = self.position_embedding_table(positions) 230 231 # Add token identity and position information. 232 x = token_embeddings + position_embeddings 233 x = self.blocks(x) 234 x = self.final_layer_norm(x) 235 logits = self.language_model_head(x) 236 237 if targets is None: 238 return logits, None 239 240 batch_size, sequence_length, vocab_size = logits.shape 241 logits_flat = logits.reshape(batch_size * sequence_length, vocab_size) 242 targets_flat = targets.reshape(batch_size * sequence_length) 243 loss = F.cross_entropy(logits_flat, targets_flat) 244 245 return logits, loss 246 247 def generate( 248 self, 249 start_ids: torch.Tensor, 250 max_new_tokens: int, 251 temperature: float, 252 generator: torch.Generator, 253 ) -> torch.Tensor: 254 """Autoregressively sample new token IDs from the model.""" 255 256 if temperature <= 0: 257 raise ValueError("temperature must be greater than zero") 258 259 generated = start_ids.clone() 260 261 for _ in range(max_new_tokens): 262 # The model can only see block_size tokens at a time. 263 context = generated[:, -self.block_size :] 264 logits, _ = self(context) 265 266 # Only the final position's logits predict the next new token. 267 next_token_logits = logits[:, -1, :] / temperature 268 probabilities = F.softmax(next_token_logits, dim=-1) 269 next_token = torch.multinomial(probabilities, num_samples=1, generator=generator) 270 generated = torch.cat((generated, next_token), dim=1) 271 272 return generated 273 274 275def load_token_ids(path: Path) -> torch.Tensor: 276 """Load one integer token ID per line into a PyTorch tensor.""" 277 278 if not path.exists(): 279 raise FileNotFoundError( 280 f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 281 ) 282 283 ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line] 284 return torch.tensor(ids, dtype=torch.long) 285 286 287def load_vocab() -> dict[str, object]: 288 """Load the character vocabulary created by the preparation script.""" 289 290 if not VOCAB_PATH.exists(): 291 raise FileNotFoundError( 292 f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 293 ) 294 295 return json.loads(VOCAB_PATH.read_text(encoding="utf-8")) 296 297 298def encode(text: str, char_to_id: dict[str, int]) -> list[int]: 299 """Convert text to character token IDs using the saved vocabulary.""" 300 301 unknown_chars = sorted(set(text) - set(char_to_id)) 302 if unknown_chars: 303 raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}") 304 305 return [char_to_id[char] for char in text] 306 307 308def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str: 309 """Convert token IDs back to text.""" 310 311 return "".join(id_to_char[token_id] for token_id in token_ids.tolist()) 312 313 314def get_batch( 315 data: torch.Tensor, 316 batch_size: int, 317 block_size: int, 318 generator: torch.Generator, 319) -> tuple[torch.Tensor, torch.Tensor]: 320 """Return a random batch of input/target blocks. 321 322 If the data is: 323 324 abcdef 325 326 and block_size is 4, one possible training example is: 327 328 input: abcd 329 target: bcde 330 331 GPT predicts the next token at every position in the input block. 332 """ 333 334 if len(data) <= block_size: 335 raise ValueError(f"Need more than {block_size} tokens, got {len(data)}") 336 337 start_positions = torch.randint( 338 high=len(data) - block_size, 339 size=(batch_size,), 340 generator=generator, 341 ) 342 343 inputs = torch.stack([data[start : start + block_size] for start in start_positions]) 344 targets = torch.stack([data[start + 1 : start + block_size + 1] for start in start_positions]) 345 346 return inputs, targets 347 348 349@torch.no_grad() 350def estimate_loss( 351 model: TinyGPTLanguageModel, 352 train_data: torch.Tensor, 353 val_data: torch.Tensor, 354 generator: torch.Generator, 355) -> dict[str, float]: 356 """Estimate train and validation loss without updating model weights.""" 357 358 model.eval() 359 losses: dict[str, float] = {} 360 361 for split_name, data in [("train", train_data), ("val", val_data)]: 362 split_losses = torch.zeros(EVAL_ITERS) 363 364 for iteration in range(EVAL_ITERS): 365 inputs, targets = get_batch(data, BATCH_SIZE, BLOCK_SIZE, generator) 366 _, loss = model(inputs, targets) 367 assert loss is not None 368 split_losses[iteration] = loss.item() 369 370 losses[split_name] = split_losses.mean().item() 371 372 model.train() 373 return losses 374 375 376def save_checkpoint( 377 model: TinyGPTLanguageModel, 378 vocab: dict[str, object], 379 final_losses: dict[str, float], 380) -> None: 381 """Save model weights and enough metadata to reload the model later.""" 382 383 CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) 384 385 checkpoint = { 386 "model_name": "tiny_gpt", 387 "model_state_dict": model.state_dict(), 388 "vocab": vocab, 389 "hyperparameters": { 390 "batch_size": BATCH_SIZE, 391 "block_size": BLOCK_SIZE, 392 "max_steps": MAX_STEPS, 393 "learning_rate": LEARNING_RATE, 394 "random_seed": RANDOM_SEED, 395 "n_embd": N_EMBD, 396 "n_head": N_HEAD, 397 "n_layer": N_LAYER, 398 "dropout": DROPOUT, 399 "sample_temperature": SAMPLE_TEMPERATURE, 400 }, 401 "final_losses": final_losses, 402 } 403 404 torch.save(checkpoint, CHECKPOINT_PATH) 405 406 407def main() -> None: 408 torch.manual_seed(RANDOM_SEED) 409 generator = torch.Generator().manual_seed(RANDOM_SEED) 410 411 train_data = load_token_ids(TRAIN_IDS_PATH) 412 val_data = load_token_ids(VAL_IDS_PATH) 413 vocab = load_vocab() 414 415 id_to_char = vocab["id_to_char"] 416 char_to_id = vocab["char_to_id"] 417 if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict): 418 raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.") 419 420 vocab_size = len(id_to_char) 421 model = TinyGPTLanguageModel( 422 vocab_size=vocab_size, 423 block_size=BLOCK_SIZE, 424 n_embd=N_EMBD, 425 n_head=N_HEAD, 426 n_layer=N_LAYER, 427 dropout=DROPOUT, 428 ) 429 optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE) 430 431 print("Training tiny GPT-style Transformer model") 432 print(f" train tokens: {len(train_data)}") 433 print(f" val tokens: {len(val_data)}") 434 print(f" vocab size: {vocab_size}") 435 print(f" block size: {BLOCK_SIZE}") 436 print(f" embedding: {N_EMBD}") 437 print(f" heads: {N_HEAD}") 438 print(f" layers: {N_LAYER}") 439 print(f" parameters: {sum(parameter.numel() for parameter in model.parameters())}") 440 print() 441 442 final_losses: dict[str, float] = {} 443 444 for step in range(MAX_STEPS + 1): 445 if step % EVAL_INTERVAL == 0: 446 losses = estimate_loss(model, train_data, val_data, generator) 447 final_losses = losses 448 print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}") 449 450 inputs, targets = get_batch(train_data, BATCH_SIZE, BLOCK_SIZE, generator) 451 _, loss = model(inputs, targets) 452 assert loss is not None 453 454 optimizer.zero_grad(set_to_none=True) 455 loss.backward() 456 optimizer.step() 457 458 save_checkpoint(model, vocab, final_losses) 459 print() 460 print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}") 461 462 start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long) 463 generated_ids = model.generate( 464 start_ids=start_ids, 465 max_new_tokens=SAMPLE_NEW_TOKENS, 466 temperature=SAMPLE_TEMPERATURE, 467 generator=generator, 468 ) 469 sample = decode(generated_ids[0], id_to_char) 470 471 print() 472 print("Sample generation:") 473 print("---") 474 print(sample) 475 print("---") 476 477 478if __name__ == "__main__": 479 main()