768
La
learning-ai-lua-code-model / files
learning-ai-lua-code-model / src / lua_code_model / sample.py
10.5 KB
1"""Generate text from a saved tiny language-model checkpoint. 2 3Training updates model weights. Inference/sampling only *uses* already-saved 4weights to predict and sample more text. 5 6Run from the project root after training at least one model: 7 8 PYTHONPATH=src uv run python -m lua_code_model.sample 9 10Examples: 11 12 PYTHONPATH=src uv run python -m lua_code_model.sample \ 13 --checkpoint checkpoints/gpt_tiny.pt \ 14 --prompt "local function" \ 15 --max-new-tokens 300 \ 16 --temperature 0.8 17 18 PYTHONPATH=src uv run python -m lua_code_model.sample \ 19 --checkpoint checkpoints/mlp_tiny.pt \ 20 --prompt "for i =" 21""" 22 23from __future__ import annotations 24 25import argparse 26from pathlib import Path 27from typing import Any 28 29import torch 30import torch.nn as nn 31import torch.nn.functional as F 32 33from lua_code_model.train_bigram import BigramLanguageModel 34from lua_code_model.train_gpt import TinyGPTLanguageModel 35from lua_code_model.train_mlp import FixedContextMLP 36 37 38PROJECT_ROOT = Path(__file__).resolve().parents[2] 39CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints" 40DEFAULT_CHECKPOINT_PATH = CHECKPOINT_DIR / "gpt_tiny.pt" 41 42DEFAULT_PROMPT = "local function" 43DEFAULT_MAX_NEW_TOKENS = 400 44DEFAULT_TEMPERATURE = 0.8 45DEFAULT_SEED = 1337 46LEFT_PAD_CHAR = "\n" 47 48 49def parse_args() -> argparse.Namespace: 50 """Read command-line options for sampling.""" 51 52 parser = argparse.ArgumentParser( 53 description="Generate text from a saved tiny Lua language-model checkpoint.", 54 ) 55 parser.add_argument( 56 "--checkpoint", 57 type=Path, 58 default=DEFAULT_CHECKPOINT_PATH, 59 help="Path to a checkpoint produced by train_bigram, train_mlp, or train_gpt.", 60 ) 61 parser.add_argument( 62 "--prompt", 63 default=DEFAULT_PROMPT, 64 help="Text the model should continue. Every character must exist in the saved vocabulary.", 65 ) 66 parser.add_argument( 67 "--max-new-tokens", 68 type=int, 69 default=DEFAULT_MAX_NEW_TOKENS, 70 help="Number of new character tokens to sample after the prompt.", 71 ) 72 parser.add_argument( 73 "--temperature", 74 type=float, 75 default=DEFAULT_TEMPERATURE, 76 help="Sampling randomness. Lower is safer/repetitive; higher is more surprising.", 77 ) 78 parser.add_argument( 79 "--seed", 80 type=int, 81 default=DEFAULT_SEED, 82 help="Random seed for repeatable sampling.", 83 ) 84 parser.add_argument( 85 "--show-metadata", 86 action="store_true", 87 help="Print checkpoint losses and hyperparameters before the generated text.", 88 ) 89 return parser.parse_args() 90 91 92def load_checkpoint(path: Path) -> dict[str, Any]: 93 """Load a checkpoint saved by one of the training scripts.""" 94 95 if not path.exists(): 96 raise FileNotFoundError( 97 f"Missing checkpoint {path}. Train a model first, for example: " 98 "PYTHONPATH=src uv run python -m lua_code_model.train_gpt" 99 ) 100 101 # The checkpoint is a local file produced by our own training scripts. It 102 # stores both tensors and small Python metadata dictionaries. 103 checkpoint = torch.load(path, map_location="cpu", weights_only=False) 104 if not isinstance(checkpoint, dict): 105 raise TypeError(f"Expected checkpoint to be a dict, got {type(checkpoint).__name__}") 106 107 return checkpoint 108 109 110def read_vocab(checkpoint: dict[str, Any]) -> tuple[list[str], dict[str, int]]: 111 """Return id->character and character->id vocabulary mappings.""" 112 113 vocab = checkpoint.get("vocab") 114 if not isinstance(vocab, dict): 115 raise TypeError("Checkpoint does not contain the expected vocabulary dictionary") 116 117 id_to_char = vocab.get("id_to_char") 118 char_to_id = vocab.get("char_to_id") 119 if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict): 120 raise TypeError("Checkpoint vocabulary has an unexpected format") 121 122 return id_to_char, char_to_id 123 124 125def encode(text: str, char_to_id: dict[str, int]) -> torch.Tensor: 126 """Convert prompt text into a 1 x prompt_length tensor of character IDs.""" 127 128 if not text: 129 raise ValueError("prompt must contain at least one character") 130 131 unknown_chars = sorted(set(text) - set(char_to_id)) 132 if unknown_chars: 133 raise ValueError( 134 "Prompt contains characters that are not in this tiny vocabulary: " 135 f"{unknown_chars}. Try a prompt using characters from the training corpus." 136 ) 137 138 token_ids = [char_to_id[char] for char in text] 139 return torch.tensor([token_ids], dtype=torch.long) 140 141 142def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str: 143 """Convert a 1D tensor of character IDs back to text.""" 144 145 return "".join(id_to_char[int(token_id)] for token_id in token_ids.tolist()) 146 147 148def require_hparam(hyperparameters: dict[str, Any], name: str) -> Any: 149 """Read a required hyperparameter with a clearer error message.""" 150 151 if name not in hyperparameters: 152 raise KeyError(f"Checkpoint is missing required hyperparameter: {name}") 153 return hyperparameters[name] 154 155 156def build_model(checkpoint: dict[str, Any], vocab_size: int) -> nn.Module: 157 """Recreate the right model architecture, then load saved weights into it.""" 158 159 model_name = checkpoint.get("model_name") 160 hyperparameters = checkpoint.get("hyperparameters", {}) 161 if not isinstance(hyperparameters, dict): 162 raise TypeError("Checkpoint hyperparameters have an unexpected format") 163 164 if model_name == "bigram": 165 model: nn.Module = BigramLanguageModel(vocab_size=vocab_size) 166 elif model_name == "fixed_context_mlp": 167 model = FixedContextMLP( 168 vocab_size=vocab_size, 169 context_size=int(require_hparam(hyperparameters, "context_size")), 170 embedding_size=int(require_hparam(hyperparameters, "embedding_size")), 171 hidden_size=int(require_hparam(hyperparameters, "hidden_size")), 172 ) 173 elif model_name == "tiny_gpt": 174 model = TinyGPTLanguageModel( 175 vocab_size=vocab_size, 176 block_size=int(require_hparam(hyperparameters, "block_size")), 177 n_embd=int(require_hparam(hyperparameters, "n_embd")), 178 n_head=int(require_hparam(hyperparameters, "n_head")), 179 n_layer=int(require_hparam(hyperparameters, "n_layer")), 180 dropout=float(require_hparam(hyperparameters, "dropout")), 181 ) 182 else: 183 raise ValueError(f"Unsupported checkpoint model_name: {model_name!r}") 184 185 state_dict = checkpoint.get("model_state_dict") 186 if not isinstance(state_dict, dict): 187 raise TypeError("Checkpoint does not contain the expected model_state_dict") 188 189 model.load_state_dict(state_dict) 190 model.eval() 191 return model 192 193 194@torch.no_grad() 195def generate_bigram( 196 model: BigramLanguageModel, 197 start_ids: torch.Tensor, 198 max_new_tokens: int, 199 temperature: float, 200 generator: torch.Generator, 201) -> torch.Tensor: 202 """Generate from a bigram checkpoint, with temperature support.""" 203 204 generated = start_ids.clone() 205 206 for _ in range(max_new_tokens): 207 last_token = generated[:, -1:] 208 logits, _ = model(last_token) 209 next_token_logits = logits[:, -1, :] / temperature 210 probabilities = F.softmax(next_token_logits, dim=-1) 211 next_token = torch.multinomial(probabilities, num_samples=1, generator=generator) 212 generated = torch.cat((generated, next_token), dim=1) 213 214 return generated 215 216 217@torch.no_grad() 218def generate_text( 219 model: nn.Module, 220 model_name: str, 221 start_ids: torch.Tensor, 222 char_to_id: dict[str, int], 223 max_new_tokens: int, 224 temperature: float, 225 generator: torch.Generator, 226) -> torch.Tensor: 227 """Dispatch to the generation method required by the saved model type.""" 228 229 if max_new_tokens < 0: 230 raise ValueError("max_new_tokens must be zero or greater") 231 if temperature <= 0: 232 raise ValueError("temperature must be greater than zero") 233 234 if model_name == "bigram": 235 if not isinstance(model, BigramLanguageModel): 236 raise TypeError("Expected BigramLanguageModel for bigram checkpoint") 237 return generate_bigram(model, start_ids, max_new_tokens, temperature, generator) 238 239 if model_name == "fixed_context_mlp": 240 if not isinstance(model, FixedContextMLP): 241 raise TypeError("Expected FixedContextMLP for fixed_context_mlp checkpoint") 242 if LEFT_PAD_CHAR not in char_to_id: 243 raise ValueError(f"Padding character {LEFT_PAD_CHAR!r} is not in the saved vocabulary") 244 return model.generate( 245 start_ids=start_ids, 246 max_new_tokens=max_new_tokens, 247 pad_token_id=char_to_id[LEFT_PAD_CHAR], 248 temperature=temperature, 249 generator=generator, 250 ) 251 252 if model_name == "tiny_gpt": 253 if not isinstance(model, TinyGPTLanguageModel): 254 raise TypeError("Expected TinyGPTLanguageModel for tiny_gpt checkpoint") 255 return model.generate( 256 start_ids=start_ids, 257 max_new_tokens=max_new_tokens, 258 temperature=temperature, 259 generator=generator, 260 ) 261 262 raise ValueError(f"Unsupported checkpoint model_name: {model_name!r}") 263 264 265def print_metadata(checkpoint: dict[str, Any]) -> None: 266 """Print useful checkpoint details for learning/debugging.""" 267 268 print("Checkpoint metadata") 269 print(f" model: {checkpoint.get('model_name')}") 270 print(f" final losses: {checkpoint.get('final_losses')}") 271 print(f" hparams: {checkpoint.get('hyperparameters')}") 272 print() 273 274 275def main() -> None: 276 args = parse_args() 277 278 torch.manual_seed(args.seed) 279 generator = torch.Generator().manual_seed(args.seed) 280 281 checkpoint = load_checkpoint(args.checkpoint) 282 id_to_char, char_to_id = read_vocab(checkpoint) 283 model_name = checkpoint.get("model_name") 284 if not isinstance(model_name, str): 285 raise TypeError("Checkpoint model_name is missing or invalid") 286 287 model = build_model(checkpoint, vocab_size=len(id_to_char)) 288 start_ids = encode(args.prompt, char_to_id) 289 generated_ids = generate_text( 290 model=model, 291 model_name=model_name, 292 start_ids=start_ids, 293 char_to_id=char_to_id, 294 max_new_tokens=args.max_new_tokens, 295 temperature=args.temperature, 296 generator=generator, 297 ) 298 generated_text = decode(generated_ids[0], id_to_char) 299 300 print(f"Loaded checkpoint: {args.checkpoint}") 301 print(f"Prompt: {args.prompt!r}") 302 print(f"Temperature: {args.temperature}") 303 print(f"New tokens: {args.max_new_tokens}") 304 print() 305 306 if args.show_metadata: 307 print_metadata(checkpoint) 308 309 print("Generated text:") 310 print("---") 311 print(generated_text) 312 print("---") 313 314 315if __name__ == "__main__": 316 main()