xonecas.io
768
La

learning-ai-lua-code-model / files

learning-ai-lua-code-model / src / lua_code_model / sample.py

10.5 KB
1"""Generate text from a saved tiny language-model checkpoint.
2
3Training updates model weights. Inference/sampling only *uses* already-saved
4weights to predict and sample more text.
5
6Run from the project root after training at least one model:
7
8    PYTHONPATH=src uv run python -m lua_code_model.sample
9
10Examples:
11
12    PYTHONPATH=src uv run python -m lua_code_model.sample \
13        --checkpoint checkpoints/gpt_tiny.pt \
14        --prompt "local function" \
15        --max-new-tokens 300 \
16        --temperature 0.8
17
18    PYTHONPATH=src uv run python -m lua_code_model.sample \
19        --checkpoint checkpoints/mlp_tiny.pt \
20        --prompt "for i ="
21"""
22
23from __future__ import annotations
24
25import argparse
26from pathlib import Path
27from typing import Any
28
29import torch
30import torch.nn as nn
31import torch.nn.functional as F
32
33from lua_code_model.train_bigram import BigramLanguageModel
34from lua_code_model.train_gpt import TinyGPTLanguageModel
35from lua_code_model.train_mlp import FixedContextMLP
36
37
38PROJECT_ROOT = Path(__file__).resolve().parents[2]
39CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
40DEFAULT_CHECKPOINT_PATH = CHECKPOINT_DIR / "gpt_tiny.pt"
41
42DEFAULT_PROMPT = "local function"
43DEFAULT_MAX_NEW_TOKENS = 400
44DEFAULT_TEMPERATURE = 0.8
45DEFAULT_SEED = 1337
46LEFT_PAD_CHAR = "\n"
47
48
49def parse_args() -> argparse.Namespace:
50    """Read command-line options for sampling."""
51
52    parser = argparse.ArgumentParser(
53        description="Generate text from a saved tiny Lua language-model checkpoint.",
54    )
55    parser.add_argument(
56        "--checkpoint",
57        type=Path,
58        default=DEFAULT_CHECKPOINT_PATH,
59        help="Path to a checkpoint produced by train_bigram, train_mlp, or train_gpt.",
60    )
61    parser.add_argument(
62        "--prompt",
63        default=DEFAULT_PROMPT,
64        help="Text the model should continue. Every character must exist in the saved vocabulary.",
65    )
66    parser.add_argument(
67        "--max-new-tokens",
68        type=int,
69        default=DEFAULT_MAX_NEW_TOKENS,
70        help="Number of new character tokens to sample after the prompt.",
71    )
72    parser.add_argument(
73        "--temperature",
74        type=float,
75        default=DEFAULT_TEMPERATURE,
76        help="Sampling randomness. Lower is safer/repetitive; higher is more surprising.",
77    )
78    parser.add_argument(
79        "--seed",
80        type=int,
81        default=DEFAULT_SEED,
82        help="Random seed for repeatable sampling.",
83    )
84    parser.add_argument(
85        "--show-metadata",
86        action="store_true",
87        help="Print checkpoint losses and hyperparameters before the generated text.",
88    )
89    return parser.parse_args()
90
91
92def load_checkpoint(path: Path) -> dict[str, Any]:
93    """Load a checkpoint saved by one of the training scripts."""
94
95    if not path.exists():
96        raise FileNotFoundError(
97            f"Missing checkpoint {path}. Train a model first, for example: "
98            "PYTHONPATH=src uv run python -m lua_code_model.train_gpt"
99        )
100
101    # The checkpoint is a local file produced by our own training scripts. It
102    # stores both tensors and small Python metadata dictionaries.
103    checkpoint = torch.load(path, map_location="cpu", weights_only=False)
104    if not isinstance(checkpoint, dict):
105        raise TypeError(f"Expected checkpoint to be a dict, got {type(checkpoint).__name__}")
106
107    return checkpoint
108
109
110def read_vocab(checkpoint: dict[str, Any]) -> tuple[list[str], dict[str, int]]:
111    """Return id->character and character->id vocabulary mappings."""
112
113    vocab = checkpoint.get("vocab")
114    if not isinstance(vocab, dict):
115        raise TypeError("Checkpoint does not contain the expected vocabulary dictionary")
116
117    id_to_char = vocab.get("id_to_char")
118    char_to_id = vocab.get("char_to_id")
119    if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict):
120        raise TypeError("Checkpoint vocabulary has an unexpected format")
121
122    return id_to_char, char_to_id
123
124
125def encode(text: str, char_to_id: dict[str, int]) -> torch.Tensor:
126    """Convert prompt text into a 1 x prompt_length tensor of character IDs."""
127
128    if not text:
129        raise ValueError("prompt must contain at least one character")
130
131    unknown_chars = sorted(set(text) - set(char_to_id))
132    if unknown_chars:
133        raise ValueError(
134            "Prompt contains characters that are not in this tiny vocabulary: "
135            f"{unknown_chars}. Try a prompt using characters from the training corpus."
136        )
137
138    token_ids = [char_to_id[char] for char in text]
139    return torch.tensor([token_ids], dtype=torch.long)
140
141
142def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str:
143    """Convert a 1D tensor of character IDs back to text."""
144
145    return "".join(id_to_char[int(token_id)] for token_id in token_ids.tolist())
146
147
148def require_hparam(hyperparameters: dict[str, Any], name: str) -> Any:
149    """Read a required hyperparameter with a clearer error message."""
150
151    if name not in hyperparameters:
152        raise KeyError(f"Checkpoint is missing required hyperparameter: {name}")
153    return hyperparameters[name]
154
155
156def build_model(checkpoint: dict[str, Any], vocab_size: int) -> nn.Module:
157    """Recreate the right model architecture, then load saved weights into it."""
158
159    model_name = checkpoint.get("model_name")
160    hyperparameters = checkpoint.get("hyperparameters", {})
161    if not isinstance(hyperparameters, dict):
162        raise TypeError("Checkpoint hyperparameters have an unexpected format")
163
164    if model_name == "bigram":
165        model: nn.Module = BigramLanguageModel(vocab_size=vocab_size)
166    elif model_name == "fixed_context_mlp":
167        model = FixedContextMLP(
168            vocab_size=vocab_size,
169            context_size=int(require_hparam(hyperparameters, "context_size")),
170            embedding_size=int(require_hparam(hyperparameters, "embedding_size")),
171            hidden_size=int(require_hparam(hyperparameters, "hidden_size")),
172        )
173    elif model_name == "tiny_gpt":
174        model = TinyGPTLanguageModel(
175            vocab_size=vocab_size,
176            block_size=int(require_hparam(hyperparameters, "block_size")),
177            n_embd=int(require_hparam(hyperparameters, "n_embd")),
178            n_head=int(require_hparam(hyperparameters, "n_head")),
179            n_layer=int(require_hparam(hyperparameters, "n_layer")),
180            dropout=float(require_hparam(hyperparameters, "dropout")),
181        )
182    else:
183        raise ValueError(f"Unsupported checkpoint model_name: {model_name!r}")
184
185    state_dict = checkpoint.get("model_state_dict")
186    if not isinstance(state_dict, dict):
187        raise TypeError("Checkpoint does not contain the expected model_state_dict")
188
189    model.load_state_dict(state_dict)
190    model.eval()
191    return model
192
193
194@torch.no_grad()
195def generate_bigram(
196    model: BigramLanguageModel,
197    start_ids: torch.Tensor,
198    max_new_tokens: int,
199    temperature: float,
200    generator: torch.Generator,
201) -> torch.Tensor:
202    """Generate from a bigram checkpoint, with temperature support."""
203
204    generated = start_ids.clone()
205
206    for _ in range(max_new_tokens):
207        last_token = generated[:, -1:]
208        logits, _ = model(last_token)
209        next_token_logits = logits[:, -1, :] / temperature
210        probabilities = F.softmax(next_token_logits, dim=-1)
211        next_token = torch.multinomial(probabilities, num_samples=1, generator=generator)
212        generated = torch.cat((generated, next_token), dim=1)
213
214    return generated
215
216
217@torch.no_grad()
218def generate_text(
219    model: nn.Module,
220    model_name: str,
221    start_ids: torch.Tensor,
222    char_to_id: dict[str, int],
223    max_new_tokens: int,
224    temperature: float,
225    generator: torch.Generator,
226) -> torch.Tensor:
227    """Dispatch to the generation method required by the saved model type."""
228
229    if max_new_tokens < 0:
230        raise ValueError("max_new_tokens must be zero or greater")
231    if temperature <= 0:
232        raise ValueError("temperature must be greater than zero")
233
234    if model_name == "bigram":
235        if not isinstance(model, BigramLanguageModel):
236            raise TypeError("Expected BigramLanguageModel for bigram checkpoint")
237        return generate_bigram(model, start_ids, max_new_tokens, temperature, generator)
238
239    if model_name == "fixed_context_mlp":
240        if not isinstance(model, FixedContextMLP):
241            raise TypeError("Expected FixedContextMLP for fixed_context_mlp checkpoint")
242        if LEFT_PAD_CHAR not in char_to_id:
243            raise ValueError(f"Padding character {LEFT_PAD_CHAR!r} is not in the saved vocabulary")
244        return model.generate(
245            start_ids=start_ids,
246            max_new_tokens=max_new_tokens,
247            pad_token_id=char_to_id[LEFT_PAD_CHAR],
248            temperature=temperature,
249            generator=generator,
250        )
251
252    if model_name == "tiny_gpt":
253        if not isinstance(model, TinyGPTLanguageModel):
254            raise TypeError("Expected TinyGPTLanguageModel for tiny_gpt checkpoint")
255        return model.generate(
256            start_ids=start_ids,
257            max_new_tokens=max_new_tokens,
258            temperature=temperature,
259            generator=generator,
260        )
261
262    raise ValueError(f"Unsupported checkpoint model_name: {model_name!r}")
263
264
265def print_metadata(checkpoint: dict[str, Any]) -> None:
266    """Print useful checkpoint details for learning/debugging."""
267
268    print("Checkpoint metadata")
269    print(f"  model:        {checkpoint.get('model_name')}")
270    print(f"  final losses: {checkpoint.get('final_losses')}")
271    print(f"  hparams:      {checkpoint.get('hyperparameters')}")
272    print()
273
274
275def main() -> None:
276    args = parse_args()
277
278    torch.manual_seed(args.seed)
279    generator = torch.Generator().manual_seed(args.seed)
280
281    checkpoint = load_checkpoint(args.checkpoint)
282    id_to_char, char_to_id = read_vocab(checkpoint)
283    model_name = checkpoint.get("model_name")
284    if not isinstance(model_name, str):
285        raise TypeError("Checkpoint model_name is missing or invalid")
286
287    model = build_model(checkpoint, vocab_size=len(id_to_char))
288    start_ids = encode(args.prompt, char_to_id)
289    generated_ids = generate_text(
290        model=model,
291        model_name=model_name,
292        start_ids=start_ids,
293        char_to_id=char_to_id,
294        max_new_tokens=args.max_new_tokens,
295        temperature=args.temperature,
296        generator=generator,
297    )
298    generated_text = decode(generated_ids[0], id_to_char)
299
300    print(f"Loaded checkpoint: {args.checkpoint}")
301    print(f"Prompt: {args.prompt!r}")
302    print(f"Temperature: {args.temperature}")
303    print(f"New tokens: {args.max_new_tokens}")
304    print()
305
306    if args.show_metadata:
307        print_metadata(checkpoint)
308
309    print("Generated text:")
310    print("---")
311    print(generated_text)
312    print("---")
313
314
315if __name__ == "__main__":
316    main()