768
La
learning-ai-lua-code-model / files
learning-ai-lua-code-model / src / lua_code_model / train_bigram.py
9.7 KB
1"""Train a tiny bigram language model with PyTorch. 2 3A language model learns to predict the next token. 4 5For this first model, the token is still just one character. The model is a 6"bigram" model because it only looks at one token of context: 7 8 current character -> predicted next character 9 10This is much simpler than GPT. GPT uses many previous tokens as context and a 11Transformer architecture. This file starts smaller so the PyTorch training loop 12is easy to study. 13 14Run from the project root after preparing the corpus: 15 16 PYTHONPATH=src uv run python -m lua_code_model.train_bigram 17""" 18 19from __future__ import annotations 20 21import json 22from pathlib import Path 23 24import torch 25import torch.nn as nn 26import torch.nn.functional as F 27 28 29PROJECT_ROOT = Path(__file__).resolve().parents[2] 30PROCESSED_DIR = PROJECT_ROOT / "data" / "processed" 31CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints" 32 33TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt" 34VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt" 35VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json" 36CHECKPOINT_PATH = CHECKPOINT_DIR / "bigram_tiny.pt" 37 38# These values are deliberately small so training is quick and inspectable. 39BATCH_SIZE = 32 40BLOCK_SIZE = 32 41MAX_STEPS = 1_000 42EVAL_INTERVAL = 100 43EVAL_ITERS = 20 44LEARNING_RATE = 0.1 45RANDOM_SEED = 1337 46SAMPLE_START_TEXT = "local" 47SAMPLE_NEW_TOKENS = 300 48 49 50class BigramLanguageModel(nn.Module): 51 """A tiny neural language model that predicts one character from one character. 52 53 The model owns one trainable table with shape: 54 55 vocab_size x vocab_size 56 57 For each input token ID, PyTorch selects one row from the table. That row is 58 a score for every possible next token. 59 60 Those raw scores are called logits. Cross-entropy loss turns the logits into 61 a training signal that says, "make the correct next token more likely." 62 """ 63 64 def __init__(self, vocab_size: int) -> None: 65 super().__init__() 66 self.token_embedding_table = nn.Embedding(vocab_size, vocab_size) 67 68 def forward( 69 self, 70 input_ids: torch.Tensor, 71 targets: torch.Tensor | None = None, 72 ) -> tuple[torch.Tensor, torch.Tensor | None]: 73 """Return logits, and optionally the loss if targets are provided. 74 75 input_ids has shape: 76 77 batch_size x block_size 78 79 logits has shape: 80 81 batch_size x block_size x vocab_size 82 83 Each position in the input predicts the next token at the same position 84 in targets. 85 """ 86 87 logits = self.token_embedding_table(input_ids) 88 89 if targets is None: 90 return logits, None 91 92 # PyTorch cross_entropy expects classes in the second dimension: 93 # logits: (examples, classes) 94 # targets: (examples,) 95 # 96 # Our logits are 3D because we have a batch of short sequences. Flatten 97 # batch and time together so every character prediction becomes one 98 # training example. 99 batch_size, block_size, vocab_size = logits.shape 100 logits_flat = logits.view(batch_size * block_size, vocab_size) 101 targets_flat = targets.view(batch_size * block_size) 102 loss = F.cross_entropy(logits_flat, targets_flat) 103 104 return logits, loss 105 106 def generate( 107 self, 108 start_ids: torch.Tensor, 109 max_new_tokens: int, 110 generator: torch.Generator, 111 ) -> torch.Tensor: 112 """Sample new token IDs from the model. 113 114 Because this is a bigram model, only the most recent token matters when 115 choosing the next token. 116 """ 117 118 generated = start_ids 119 120 for _ in range(max_new_tokens): 121 last_token = generated[:, -1:] 122 logits, _ = self(last_token) 123 next_token_logits = logits[:, -1, :] 124 probabilities = F.softmax(next_token_logits, dim=-1) 125 next_token = torch.multinomial(probabilities, num_samples=1, generator=generator) 126 generated = torch.cat((generated, next_token), dim=1) 127 128 return generated 129 130 131def load_token_ids(path: Path) -> torch.Tensor: 132 """Load one integer token ID per line into a PyTorch tensor.""" 133 134 if not path.exists(): 135 raise FileNotFoundError( 136 f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 137 ) 138 139 ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line] 140 return torch.tensor(ids, dtype=torch.long) 141 142 143def load_vocab() -> dict[str, object]: 144 """Load the character vocabulary created by the preparation script.""" 145 146 if not VOCAB_PATH.exists(): 147 raise FileNotFoundError( 148 f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus" 149 ) 150 151 return json.loads(VOCAB_PATH.read_text(encoding="utf-8")) 152 153 154def encode(text: str, char_to_id: dict[str, int]) -> list[int]: 155 """Convert text to character token IDs using the saved vocabulary.""" 156 157 unknown_chars = sorted(set(text) - set(char_to_id)) 158 if unknown_chars: 159 raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}") 160 161 return [char_to_id[char] for char in text] 162 163 164def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str: 165 """Convert token IDs back to text.""" 166 167 return "".join(id_to_char[token_id] for token_id in token_ids.tolist()) 168 169 170def get_batch( 171 data: torch.Tensor, 172 batch_size: int, 173 block_size: int, 174 generator: torch.Generator, 175) -> tuple[torch.Tensor, torch.Tensor]: 176 """Return a random batch of input/target examples. 177 178 If the data is: 179 180 abcdef 181 182 and block_size is 4, one possible training example is: 183 184 input: abcd 185 target: bcde 186 187 The target is shifted one character to the right because the task is next 188 character prediction. 189 """ 190 191 if len(data) <= block_size: 192 raise ValueError(f"Need more than {block_size} tokens, got {len(data)}") 193 194 start_positions = torch.randint( 195 high=len(data) - block_size, 196 size=(batch_size,), 197 generator=generator, 198 ) 199 200 inputs = torch.stack([data[start : start + block_size] for start in start_positions]) 201 targets = torch.stack([data[start + 1 : start + block_size + 1] for start in start_positions]) 202 203 return inputs, targets 204 205 206@torch.no_grad() 207def estimate_loss( 208 model: BigramLanguageModel, 209 train_data: torch.Tensor, 210 val_data: torch.Tensor, 211 generator: torch.Generator, 212) -> dict[str, float]: 213 """Estimate train and validation loss without updating model weights.""" 214 215 model.eval() 216 losses: dict[str, float] = {} 217 218 for split_name, data in [("train", train_data), ("val", val_data)]: 219 split_losses = torch.zeros(EVAL_ITERS) 220 221 for iteration in range(EVAL_ITERS): 222 inputs, targets = get_batch(data, BATCH_SIZE, BLOCK_SIZE, generator) 223 _, loss = model(inputs, targets) 224 assert loss is not None 225 split_losses[iteration] = loss.item() 226 227 losses[split_name] = split_losses.mean().item() 228 229 model.train() 230 return losses 231 232 233def save_checkpoint( 234 model: BigramLanguageModel, 235 vocab: dict[str, object], 236 final_losses: dict[str, float], 237) -> None: 238 """Save model weights and enough metadata to reload the model later.""" 239 240 CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) 241 242 checkpoint = { 243 "model_name": "bigram", 244 "model_state_dict": model.state_dict(), 245 "vocab": vocab, 246 "hyperparameters": { 247 "batch_size": BATCH_SIZE, 248 "block_size": BLOCK_SIZE, 249 "max_steps": MAX_STEPS, 250 "learning_rate": LEARNING_RATE, 251 "random_seed": RANDOM_SEED, 252 }, 253 "final_losses": final_losses, 254 } 255 256 torch.save(checkpoint, CHECKPOINT_PATH) 257 258 259def main() -> None: 260 torch.manual_seed(RANDOM_SEED) 261 generator = torch.Generator().manual_seed(RANDOM_SEED) 262 263 train_data = load_token_ids(TRAIN_IDS_PATH) 264 val_data = load_token_ids(VAL_IDS_PATH) 265 vocab = load_vocab() 266 267 id_to_char = vocab["id_to_char"] 268 char_to_id = vocab["char_to_id"] 269 if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict): 270 raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.") 271 272 vocab_size = len(id_to_char) 273 model = BigramLanguageModel(vocab_size) 274 optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE) 275 276 print("Training tiny bigram model") 277 print(f" train tokens: {len(train_data)}") 278 print(f" val tokens: {len(val_data)}") 279 print(f" vocab size: {vocab_size}") 280 print(f" parameters: {sum(parameter.numel() for parameter in model.parameters())}") 281 print() 282 283 final_losses: dict[str, float] = {} 284 285 for step in range(MAX_STEPS + 1): 286 if step % EVAL_INTERVAL == 0: 287 losses = estimate_loss(model, train_data, val_data, generator) 288 final_losses = losses 289 print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}") 290 291 inputs, targets = get_batch(train_data, BATCH_SIZE, BLOCK_SIZE, generator) 292 _, loss = model(inputs, targets) 293 assert loss is not None 294 295 # Standard PyTorch training pattern: 296 # 1. clear old gradients 297 # 2. compute new gradients with backpropagation 298 # 3. update parameters with the optimizer 299 optimizer.zero_grad(set_to_none=True) 300 loss.backward() 301 optimizer.step() 302 303 save_checkpoint(model, vocab, final_losses) 304 print() 305 print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}") 306 307 start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long) 308 generated_ids = model.generate(start_ids, SAMPLE_NEW_TOKENS, generator) 309 sample = decode(generated_ids[0], id_to_char) 310 311 print() 312 print("Sample generation:") 313 print("---") 314 print(sample) 315 print("---") 316 317 318if __name__ == "__main__": 319 main()