768
La
learning-ai-lua-code-model / files
learning-ai-lua-code-model / src / lua_code_model / train_mlp.py
11.5 KB
1"""Train a tiny fixed-context MLP language model with PyTorch. 2 3This is the next step after the bigram model. 4 5The bigram model saw only one character: 6 7 current character -> next character 8 9This MLP model sees a fixed window of previous characters: 10 11 previous 32 characters -> next character 12 13That makes it smarter than the bigram model, but it is still much simpler than 14GPT. GPT also uses token embeddings, but replaces this simple MLP with a 15Transformer that can decide which previous tokens to pay attention to. 16 17Run from the project root after preparing the corpus: 18 19 PYTHONPATH=src uv run python -m lua_code_model.train_mlp 20""" 21 22from __future__ import annotations 23 24import json 25from pathlib import Path 26 27import torch 28import torch.nn as nn 29import torch.nn.functional as F 30 31 32PROJECT_ROOT = Path(__file__).resolve().parents[2] 33PROCESSED_DIR = PROJECT_ROOT / "data" / "processed" 34CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints" 35 36TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt" 37VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt" 38VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json" 39CHECKPOINT_PATH = CHECKPOINT_DIR / "mlp_tiny.pt" 40 41# Model/training settings. These are intentionally small and easy to change. 42CONTEXT_SIZE = 32 43EMBEDDING_SIZE = 32 44HIDDEN_SIZE = 128 45BATCH_SIZE = 32 46MAX_STEPS = 2_000 47EVAL_INTERVAL = 200 48EVAL_ITERS = 20 49LEARNING_RATE = 3e-3 50RANDOM_SEED = 1337 51 52# This prompt appears in the tiny corpus, so the model has a fair chance to 53# continue it. Since this is a tiny overfitting experiment, that is okay. 54SAMPLE_START_TEXT = "local function count_items(items)" 55SAMPLE_NEW_TOKENS = 400 56SAMPLE_TEMPERATURE = 0.8 57LEFT_PAD_CHAR = "\n" 58 59 60class FixedContextMLP(nn.Module): 61 """A tiny neural language model with a fixed-size context window. 62 63 For each example, the model receives CONTEXT_SIZE token IDs. It then: 64 65 1. looks up an embedding vector for each token 66 2. flattens those vectors into one long vector 67 3. passes that vector through a hidden layer and ReLU activation 68 4. returns logits for the next token 69 70 The key limitation: the context size is fixed. If CONTEXT_SIZE is 32, the 71 model can only use the previous 32 characters to predict the next one. 72 """ 73 74 def __init__( 75 self, 76 vocab_size: int, 77 context_size: int, 78 embedding_size: int, 79 hidden_size: int, 80 ) -> None: 81 super().__init__() 82 self.context_size = context_size 83 self.embedding_size = embedding_size 84 85 self.token_embedding_table = nn.Embedding(vocab_size, embedding_size) 86 self.network = nn.Sequential( 87 nn.Linear(context_size * embedding_size, hidden_size), 88 nn.ReLU(), 89 nn.Linear(hidden_size, vocab_size), 90 ) 91 92 def forward( 93 self, 94 input_ids: torch.Tensor, 95 targets: torch.Tensor | None = None, 96 ) -> tuple[torch.Tensor, torch.Tensor | None]: 97 """Return logits, and optionally the loss if targets are provided. 98 99 input_ids shape: 100 101 batch_size x context_size 102 103 logits shape: 104 105 batch_size x vocab_size 106 107 targets shape: 108 109 batch_size 110 111 Unlike the bigram script, this model makes one prediction per context 112 window, not one prediction per position inside the window. 113 """ 114 115 embeddings = self.token_embedding_table(input_ids) 116 batch_size = embeddings.shape[0] 117 118 # embeddings starts as: batch_size x context_size x embedding_size 119 # The MLP expects: batch_size x flattened_features 120 flattened = embeddings.reshape(batch_size, self.context_size * self.embedding_size) 121 logits = self.network(flattened) 122 123 if targets is None: 124 return logits, None 125 126 loss = F.cross_entropy(logits, targets) 127 return logits, loss 128 129 def generate( 130 self, 131 start_ids: torch.Tensor, 132 max_new_tokens: int, 133 pad_token_id: int, 134 temperature: float, 135 generator: torch.Generator, 136 ) -> torch.Tensor: 137 """Sample new token IDs from the model. 138 139 Generation is autoregressive: 140 141 1. use the current text as context 142 2. predict the next token 143 3. append that token 144 4. repeat 145 146 If the prompt is shorter than context_size, we left-pad it with newlines 147 so the model still receives exactly context_size tokens. 148 """ 149 150 if temperature <= 0: 151 raise ValueError("temperature must be greater than zero") 152 153 generated = start_ids.clone() 154 155 for _ in range(max_new_tokens): 156 if generated.shape[1] < self.context_size: 157 pad_width = self.context_size - generated.shape[1] 158 padding = torch.full( 159 (generated.shape[0], pad_width), 160 fill_value=pad_token_id, 161 dtype=torch.long, 162 ) 163 context = torch.cat((padding, generated), dim=1) 164 else: 165 context = generated[:, -self.context_size :] 166 167 logits, _ = self(context) 168 probabilities = F.softmax(logits / temperature, dim=-1) 169 next_token = torch.multinomial(probabilities, num_samples=1, generator=generator) 170 generated = torch.cat((generated, next_token), dim=1) 171 172 return generated 173 174 175def load_token_ids(path: Path) -> torch.Tensor: 176 """Load one integer token ID per line into a PyTorch tensor.""" 177 178 if not path.exists(): 179 raise FileNotFoundError( 180 f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 181 ) 182 183 ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line] 184 return torch.tensor(ids, dtype=torch.long) 185 186 187def load_vocab() -> dict[str, object]: 188 """Load the character vocabulary created by the preparation script.""" 189 190 if not VOCAB_PATH.exists(): 191 raise FileNotFoundError( 192 f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 193 ) 194 195 return json.loads(VOCAB_PATH.read_text(encoding="utf-8")) 196 197 198def encode(text: str, char_to_id: dict[str, int]) -> list[int]: 199 """Convert text to character token IDs using the saved vocabulary.""" 200 201 unknown_chars = sorted(set(text) - set(char_to_id)) 202 if unknown_chars: 203 raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}") 204 205 return [char_to_id[char] for char in text] 206 207 208def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str: 209 """Convert token IDs back to text.""" 210 211 return "".join(id_to_char[token_id] for token_id in token_ids.tolist()) 212 213 214def get_batch( 215 data: torch.Tensor, 216 batch_size: int, 217 context_size: int, 218 generator: torch.Generator, 219) -> tuple[torch.Tensor, torch.Tensor]: 220 """Return random fixed-context examples. 221 222 If the data is: 223 224 abcdef 225 226 and context_size is 4, one possible training example is: 227 228 input: abcd 229 target: e 230 231 The target is one token, immediately after the context window. 232 """ 233 234 if len(data) <= context_size: 235 raise ValueError(f"Need more than {context_size} tokens, got {len(data)}") 236 237 start_positions = torch.randint( 238 high=len(data) - context_size, 239 size=(batch_size,), 240 generator=generator, 241 ) 242 243 inputs = torch.stack([data[start : start + context_size] for start in start_positions]) 244 targets = torch.stack([data[start + context_size] for start in start_positions]) 245 246 return inputs, targets 247 248 249@torch.no_grad() 250def estimate_loss( 251 model: FixedContextMLP, 252 train_data: torch.Tensor, 253 val_data: torch.Tensor, 254 generator: torch.Generator, 255) -> dict[str, float]: 256 """Estimate train and validation loss without updating model weights.""" 257 258 model.eval() 259 losses: dict[str, float] = {} 260 261 for split_name, data in [("train", train_data), ("val", val_data)]: 262 split_losses = torch.zeros(EVAL_ITERS) 263 264 for iteration in range(EVAL_ITERS): 265 inputs, targets = get_batch(data, BATCH_SIZE, CONTEXT_SIZE, generator) 266 _, loss = model(inputs, targets) 267 assert loss is not None 268 split_losses[iteration] = loss.item() 269 270 losses[split_name] = split_losses.mean().item() 271 272 model.train() 273 return losses 274 275 276def save_checkpoint( 277 model: FixedContextMLP, 278 vocab: dict[str, object], 279 final_losses: dict[str, float], 280) -> None: 281 """Save model weights and enough metadata to reload the model later.""" 282 283 CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) 284 285 checkpoint = { 286 "model_name": "fixed_context_mlp", 287 "model_state_dict": model.state_dict(), 288 "vocab": vocab, 289 "hyperparameters": { 290 "context_size": CONTEXT_SIZE, 291 "embedding_size": EMBEDDING_SIZE, 292 "hidden_size": HIDDEN_SIZE, 293 "batch_size": BATCH_SIZE, 294 "max_steps": MAX_STEPS, 295 "learning_rate": LEARNING_RATE, 296 "random_seed": RANDOM_SEED, 297 "sample_temperature": SAMPLE_TEMPERATURE, 298 }, 299 "final_losses": final_losses, 300 } 301 302 torch.save(checkpoint, CHECKPOINT_PATH) 303 304 305def main() -> None: 306 torch.manual_seed(RANDOM_SEED) 307 generator = torch.Generator().manual_seed(RANDOM_SEED) 308 309 train_data = load_token_ids(TRAIN_IDS_PATH) 310 val_data = load_token_ids(VAL_IDS_PATH) 311 vocab = load_vocab() 312 313 id_to_char = vocab["id_to_char"] 314 char_to_id = vocab["char_to_id"] 315 if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict): 316 raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.") 317 318 if LEFT_PAD_CHAR not in char_to_id: 319 raise ValueError(f"LEFT_PAD_CHAR={LEFT_PAD_CHAR!r} is not in the vocabulary") 320 321 vocab_size = len(id_to_char) 322 model = FixedContextMLP( 323 vocab_size=vocab_size, 324 context_size=CONTEXT_SIZE, 325 embedding_size=EMBEDDING_SIZE, 326 hidden_size=HIDDEN_SIZE, 327 ) 328 optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE) 329 330 print("Training fixed-context MLP model") 331 print(f" train tokens: {len(train_data)}") 332 print(f" val tokens: {len(val_data)}") 333 print(f" vocab size: {vocab_size}") 334 print(f" context size: {CONTEXT_SIZE}") 335 print(f" embedding size: {EMBEDDING_SIZE}") 336 print(f" hidden size: {HIDDEN_SIZE}") 337 print(f" parameters: {sum(parameter.numel() for parameter in model.parameters())}") 338 print() 339 340 final_losses: dict[str, float] = {} 341 342 for step in range(MAX_STEPS + 1): 343 if step % EVAL_INTERVAL == 0: 344 losses = estimate_loss(model, train_data, val_data, generator) 345 final_losses = losses 346 print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}") 347 348 inputs, targets = get_batch(train_data, BATCH_SIZE, CONTEXT_SIZE, generator) 349 _, loss = model(inputs, targets) 350 assert loss is not None 351 352 optimizer.zero_grad(set_to_none=True) 353 loss.backward() 354 optimizer.step() 355 356 save_checkpoint(model, vocab, final_losses) 357 print() 358 print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}") 359 360 start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long) 361 pad_token_id = char_to_id[LEFT_PAD_CHAR] 362 generated_ids = model.generate( 363 start_ids=start_ids, 364 max_new_tokens=SAMPLE_NEW_TOKENS, 365 pad_token_id=pad_token_id, 366 temperature=SAMPLE_TEMPERATURE, 367 generator=generator, 368 ) 369 sample = decode(generated_ids[0], id_to_char) 370 371 print() 372 print("Sample generation:") 373 print("---") 374 print(sample) 375 print("---") 376 377 378if __name__ == "__main__": 379 main()