xonecas.io
768
La

learning-ai-lua-code-model / files

learning-ai-lua-code-model / src / lua_code_model / train_mlp.py

11.5 KB
1"""Train a tiny fixed-context MLP language model with PyTorch.
2
3This is the next step after the bigram model.
4
5The bigram model saw only one character:
6
7    current character -> next character
8
9This MLP model sees a fixed window of previous characters:
10
11    previous 32 characters -> next character
12
13That makes it smarter than the bigram model, but it is still much simpler than
14GPT. GPT also uses token embeddings, but replaces this simple MLP with a
15Transformer that can decide which previous tokens to pay attention to.
16
17Run from the project root after preparing the corpus:
18
19    PYTHONPATH=src uv run python -m lua_code_model.train_mlp
20"""
21
22from __future__ import annotations
23
24import json
25from pathlib import Path
26
27import torch
28import torch.nn as nn
29import torch.nn.functional as F
30
31
32PROJECT_ROOT = Path(__file__).resolve().parents[2]
33PROCESSED_DIR = PROJECT_ROOT / "data" / "processed"
34CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
35
36TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt"
37VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt"
38VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json"
39CHECKPOINT_PATH = CHECKPOINT_DIR / "mlp_tiny.pt"
40
41# Model/training settings. These are intentionally small and easy to change.
42CONTEXT_SIZE = 32
43EMBEDDING_SIZE = 32
44HIDDEN_SIZE = 128
45BATCH_SIZE = 32
46MAX_STEPS = 2_000
47EVAL_INTERVAL = 200
48EVAL_ITERS = 20
49LEARNING_RATE = 3e-3
50RANDOM_SEED = 1337
51
52# This prompt appears in the tiny corpus, so the model has a fair chance to
53# continue it. Since this is a tiny overfitting experiment, that is okay.
54SAMPLE_START_TEXT = "local function count_items(items)"
55SAMPLE_NEW_TOKENS = 400
56SAMPLE_TEMPERATURE = 0.8
57LEFT_PAD_CHAR = "\n"
58
59
60class FixedContextMLP(nn.Module):
61    """A tiny neural language model with a fixed-size context window.
62
63    For each example, the model receives CONTEXT_SIZE token IDs. It then:
64
65    1. looks up an embedding vector for each token
66    2. flattens those vectors into one long vector
67    3. passes that vector through a hidden layer and ReLU activation
68    4. returns logits for the next token
69
70    The key limitation: the context size is fixed. If CONTEXT_SIZE is 32, the
71    model can only use the previous 32 characters to predict the next one.
72    """
73
74    def __init__(
75        self,
76        vocab_size: int,
77        context_size: int,
78        embedding_size: int,
79        hidden_size: int,
80    ) -> None:
81        super().__init__()
82        self.context_size = context_size
83        self.embedding_size = embedding_size
84
85        self.token_embedding_table = nn.Embedding(vocab_size, embedding_size)
86        self.network = nn.Sequential(
87            nn.Linear(context_size * embedding_size, hidden_size),
88            nn.ReLU(),
89            nn.Linear(hidden_size, vocab_size),
90        )
91
92    def forward(
93        self,
94        input_ids: torch.Tensor,
95        targets: torch.Tensor | None = None,
96    ) -> tuple[torch.Tensor, torch.Tensor | None]:
97        """Return logits, and optionally the loss if targets are provided.
98
99        input_ids shape:
100
101            batch_size x context_size
102
103        logits shape:
104
105            batch_size x vocab_size
106
107        targets shape:
108
109            batch_size
110
111        Unlike the bigram script, this model makes one prediction per context
112        window, not one prediction per position inside the window.
113        """
114
115        embeddings = self.token_embedding_table(input_ids)
116        batch_size = embeddings.shape[0]
117
118        # embeddings starts as: batch_size x context_size x embedding_size
119        # The MLP expects:      batch_size x flattened_features
120        flattened = embeddings.reshape(batch_size, self.context_size * self.embedding_size)
121        logits = self.network(flattened)
122
123        if targets is None:
124            return logits, None
125
126        loss = F.cross_entropy(logits, targets)
127        return logits, loss
128
129    def generate(
130        self,
131        start_ids: torch.Tensor,
132        max_new_tokens: int,
133        pad_token_id: int,
134        temperature: float,
135        generator: torch.Generator,
136    ) -> torch.Tensor:
137        """Sample new token IDs from the model.
138
139        Generation is autoregressive:
140
141        1. use the current text as context
142        2. predict the next token
143        3. append that token
144        4. repeat
145
146        If the prompt is shorter than context_size, we left-pad it with newlines
147        so the model still receives exactly context_size tokens.
148        """
149
150        if temperature <= 0:
151            raise ValueError("temperature must be greater than zero")
152
153        generated = start_ids.clone()
154
155        for _ in range(max_new_tokens):
156            if generated.shape[1] < self.context_size:
157                pad_width = self.context_size - generated.shape[1]
158                padding = torch.full(
159                    (generated.shape[0], pad_width),
160                    fill_value=pad_token_id,
161                    dtype=torch.long,
162                )
163                context = torch.cat((padding, generated), dim=1)
164            else:
165                context = generated[:, -self.context_size :]
166
167            logits, _ = self(context)
168            probabilities = F.softmax(logits / temperature, dim=-1)
169            next_token = torch.multinomial(probabilities, num_samples=1, generator=generator)
170            generated = torch.cat((generated, next_token), dim=1)
171
172        return generated
173
174
175def load_token_ids(path: Path) -> torch.Tensor:
176    """Load one integer token ID per line into a PyTorch tensor."""
177
178    if not path.exists():
179        raise FileNotFoundError(
180            f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
181        )
182
183    ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
184    return torch.tensor(ids, dtype=torch.long)
185
186
187def load_vocab() -> dict[str, object]:
188    """Load the character vocabulary created by the preparation script."""
189
190    if not VOCAB_PATH.exists():
191        raise FileNotFoundError(
192            f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
193        )
194
195    return json.loads(VOCAB_PATH.read_text(encoding="utf-8"))
196
197
198def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
199    """Convert text to character token IDs using the saved vocabulary."""
200
201    unknown_chars = sorted(set(text) - set(char_to_id))
202    if unknown_chars:
203        raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}")
204
205    return [char_to_id[char] for char in text]
206
207
208def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str:
209    """Convert token IDs back to text."""
210
211    return "".join(id_to_char[token_id] for token_id in token_ids.tolist())
212
213
214def get_batch(
215    data: torch.Tensor,
216    batch_size: int,
217    context_size: int,
218    generator: torch.Generator,
219) -> tuple[torch.Tensor, torch.Tensor]:
220    """Return random fixed-context examples.
221
222    If the data is:
223
224        abcdef
225
226    and context_size is 4, one possible training example is:
227
228        input:  abcd
229        target: e
230
231    The target is one token, immediately after the context window.
232    """
233
234    if len(data) <= context_size:
235        raise ValueError(f"Need more than {context_size} tokens, got {len(data)}")
236
237    start_positions = torch.randint(
238        high=len(data) - context_size,
239        size=(batch_size,),
240        generator=generator,
241    )
242
243    inputs = torch.stack([data[start : start + context_size] for start in start_positions])
244    targets = torch.stack([data[start + context_size] for start in start_positions])
245
246    return inputs, targets
247
248
249@torch.no_grad()
250def estimate_loss(
251    model: FixedContextMLP,
252    train_data: torch.Tensor,
253    val_data: torch.Tensor,
254    generator: torch.Generator,
255) -> dict[str, float]:
256    """Estimate train and validation loss without updating model weights."""
257
258    model.eval()
259    losses: dict[str, float] = {}
260
261    for split_name, data in [("train", train_data), ("val", val_data)]:
262        split_losses = torch.zeros(EVAL_ITERS)
263
264        for iteration in range(EVAL_ITERS):
265            inputs, targets = get_batch(data, BATCH_SIZE, CONTEXT_SIZE, generator)
266            _, loss = model(inputs, targets)
267            assert loss is not None
268            split_losses[iteration] = loss.item()
269
270        losses[split_name] = split_losses.mean().item()
271
272    model.train()
273    return losses
274
275
276def save_checkpoint(
277    model: FixedContextMLP,
278    vocab: dict[str, object],
279    final_losses: dict[str, float],
280) -> None:
281    """Save model weights and enough metadata to reload the model later."""
282
283    CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
284
285    checkpoint = {
286        "model_name": "fixed_context_mlp",
287        "model_state_dict": model.state_dict(),
288        "vocab": vocab,
289        "hyperparameters": {
290            "context_size": CONTEXT_SIZE,
291            "embedding_size": EMBEDDING_SIZE,
292            "hidden_size": HIDDEN_SIZE,
293            "batch_size": BATCH_SIZE,
294            "max_steps": MAX_STEPS,
295            "learning_rate": LEARNING_RATE,
296            "random_seed": RANDOM_SEED,
297            "sample_temperature": SAMPLE_TEMPERATURE,
298        },
299        "final_losses": final_losses,
300    }
301
302    torch.save(checkpoint, CHECKPOINT_PATH)
303
304
305def main() -> None:
306    torch.manual_seed(RANDOM_SEED)
307    generator = torch.Generator().manual_seed(RANDOM_SEED)
308
309    train_data = load_token_ids(TRAIN_IDS_PATH)
310    val_data = load_token_ids(VAL_IDS_PATH)
311    vocab = load_vocab()
312
313    id_to_char = vocab["id_to_char"]
314    char_to_id = vocab["char_to_id"]
315    if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict):
316        raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.")
317
318    if LEFT_PAD_CHAR not in char_to_id:
319        raise ValueError(f"LEFT_PAD_CHAR={LEFT_PAD_CHAR!r} is not in the vocabulary")
320
321    vocab_size = len(id_to_char)
322    model = FixedContextMLP(
323        vocab_size=vocab_size,
324        context_size=CONTEXT_SIZE,
325        embedding_size=EMBEDDING_SIZE,
326        hidden_size=HIDDEN_SIZE,
327    )
328    optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)
329
330    print("Training fixed-context MLP model")
331    print(f"  train tokens:    {len(train_data)}")
332    print(f"  val tokens:      {len(val_data)}")
333    print(f"  vocab size:      {vocab_size}")
334    print(f"  context size:    {CONTEXT_SIZE}")
335    print(f"  embedding size:  {EMBEDDING_SIZE}")
336    print(f"  hidden size:     {HIDDEN_SIZE}")
337    print(f"  parameters:      {sum(parameter.numel() for parameter in model.parameters())}")
338    print()
339
340    final_losses: dict[str, float] = {}
341
342    for step in range(MAX_STEPS + 1):
343        if step % EVAL_INTERVAL == 0:
344            losses = estimate_loss(model, train_data, val_data, generator)
345            final_losses = losses
346            print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
347
348        inputs, targets = get_batch(train_data, BATCH_SIZE, CONTEXT_SIZE, generator)
349        _, loss = model(inputs, targets)
350        assert loss is not None
351
352        optimizer.zero_grad(set_to_none=True)
353        loss.backward()
354        optimizer.step()
355
356    save_checkpoint(model, vocab, final_losses)
357    print()
358    print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}")
359
360    start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long)
361    pad_token_id = char_to_id[LEFT_PAD_CHAR]
362    generated_ids = model.generate(
363        start_ids=start_ids,
364        max_new_tokens=SAMPLE_NEW_TOKENS,
365        pad_token_id=pad_token_id,
366        temperature=SAMPLE_TEMPERATURE,
367        generator=generator,
368    )
369    sample = decode(generated_ids[0], id_to_char)
370
371    print()
372    print("Sample generation:")
373    print("---")
374    print(sample)
375    print("---")
376
377
378if __name__ == "__main__":
379    main()