@@ -0,0 +1,379 @@
1+"""Train a tiny fixed-context MLP language model with PyTorch.
2+
3+This is the next step after the bigram model.
4+
5+The bigram model saw only one character:
6+
7+ current character -> next character
8+
9+This MLP model sees a fixed window of previous characters:
10+
11+ previous 32 characters -> next character
12+
13+That makes it smarter than the bigram model, but it is still much simpler than
14+GPT. GPT also uses token embeddings, but replaces this simple MLP with a
15+Transformer that can decide which previous tokens to pay attention to.
16+
17+Run from the project root after preparing the corpus:
18+
19+ PYTHONPATH=src uv run python -m lua_code_model.train_mlp
20+"""
21+
22+from __future__ import annotations
23+
24+import json
25+from pathlib import Path
26+
27+import torch
28+import torch.nn as nn
29+import torch.nn.functional as F
30+
31+
32+PROJECT_ROOT = Path(__file__).resolve().parents[2]
33+PROCESSED_DIR = PROJECT_ROOT / "data" / "processed"
34+CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
35+
36+TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt"
37+VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt"
38+VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json"
39+CHECKPOINT_PATH = CHECKPOINT_DIR / "mlp_tiny.pt"
40+
41+# Model/training settings. These are intentionally small and easy to change.
42+CONTEXT_SIZE = 32
43+EMBEDDING_SIZE = 32
44+HIDDEN_SIZE = 128
45+BATCH_SIZE = 32
46+MAX_STEPS = 2_000
47+EVAL_INTERVAL = 200
48+EVAL_ITERS = 20
49+LEARNING_RATE = 3e-3
50+RANDOM_SEED = 1337
51+
52+# This prompt appears in the tiny corpus, so the model has a fair chance to
53+# continue it. Since this is a tiny overfitting experiment, that is okay.
54+SAMPLE_START_TEXT = "local function count_items(items)"
55+SAMPLE_NEW_TOKENS = 400
56+SAMPLE_TEMPERATURE = 0.8
57+LEFT_PAD_CHAR = "\n"
58+
59+
60+class FixedContextMLP(nn.Module):
61+ """A tiny neural language model with a fixed-size context window.
62+
63+ For each example, the model receives CONTEXT_SIZE token IDs. It then:
64+
65+ 1. looks up an embedding vector for each token
66+ 2. flattens those vectors into one long vector
67+ 3. passes that vector through a hidden layer and ReLU activation
68+ 4. returns logits for the next token
69+
70+ The key limitation: the context size is fixed. If CONTEXT_SIZE is 32, the
71+ model can only use the previous 32 characters to predict the next one.
72+ """
73+
74+ def __init__(
75+ self,
76+ vocab_size: int,
77+ context_size: int,
78+ embedding_size: int,
79+ hidden_size: int,
80+ ) -> None:
81+ super().__init__()
82+ self.context_size = context_size
83+ self.embedding_size = embedding_size
84+
85+ self.token_embedding_table = nn.Embedding(vocab_size, embedding_size)
86+ self.network = nn.Sequential(
87+ nn.Linear(context_size * embedding_size, hidden_size),
88+ nn.ReLU(),
89+ nn.Linear(hidden_size, vocab_size),
90+ )
91+
92+ def forward(
93+ self,
94+ input_ids: torch.Tensor,
95+ targets: torch.Tensor | None = None,
96+ ) -> tuple[torch.Tensor, torch.Tensor | None]:
97+ """Return logits, and optionally the loss if targets are provided.
98+
99+ input_ids shape:
100+
101+ batch_size x context_size
102+
103+ logits shape:
104+
105+ batch_size x vocab_size
106+
107+ targets shape:
108+
109+ batch_size
110+
111+ Unlike the bigram script, this model makes one prediction per context
112+ window, not one prediction per position inside the window.
113+ """
114+
115+ embeddings = self.token_embedding_table(input_ids)
116+ batch_size = embeddings.shape[0]
117+
118+ # embeddings starts as: batch_size x context_size x embedding_size
119+ # The MLP expects: batch_size x flattened_features
120+ flattened = embeddings.reshape(batch_size, self.context_size * self.embedding_size)
121+ logits = self.network(flattened)
122+
123+ if targets is None:
124+ return logits, None
125+
126+ loss = F.cross_entropy(logits, targets)
127+ return logits, loss
128+
129+ def generate(
130+ self,
131+ start_ids: torch.Tensor,
132+ max_new_tokens: int,
133+ pad_token_id: int,
134+ temperature: float,
135+ generator: torch.Generator,
136+ ) -> torch.Tensor:
137+ """Sample new token IDs from the model.
138+
139+ Generation is autoregressive:
140+
141+ 1. use the current text as context
142+ 2. predict the next token
143+ 3. append that token
144+ 4. repeat
145+
146+ If the prompt is shorter than context_size, we left-pad it with newlines
147+ so the model still receives exactly context_size tokens.
148+ """
149+
150+ if temperature <= 0:
151+ raise ValueError("temperature must be greater than zero")
152+
153+ generated = start_ids.clone()
154+
155+ for _ in range(max_new_tokens):
156+ if generated.shape[1] < self.context_size:
157+ pad_width = self.context_size - generated.shape[1]
158+ padding = torch.full(
159+ (generated.shape[0], pad_width),
160+ fill_value=pad_token_id,
161+ dtype=torch.long,
162+ )
163+ context = torch.cat((padding, generated), dim=1)
164+ else:
165+ context = generated[:, -self.context_size :]
166+
167+ logits, _ = self(context)
168+ probabilities = F.softmax(logits / temperature, dim=-1)
169+ next_token = torch.multinomial(probabilities, num_samples=1, generator=generator)
170+ generated = torch.cat((generated, next_token), dim=1)
171+
172+ return generated
173+
174+
175+def load_token_ids(path: Path) -> torch.Tensor:
176+ """Load one integer token ID per line into a PyTorch tensor."""
177+
178+ if not path.exists():
179+ raise FileNotFoundError(
180+ f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
181+ )
182+
183+ ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
184+ return torch.tensor(ids, dtype=torch.long)
185+
186+
187+def load_vocab() -> dict[str, object]:
188+ """Load the character vocabulary created by the preparation script."""
189+
190+ if not VOCAB_PATH.exists():
191+ raise FileNotFoundError(
192+ f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
193+ )
194+
195+ return json.loads(VOCAB_PATH.read_text(encoding="utf-8"))
196+
197+
198+def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
199+ """Convert text to character token IDs using the saved vocabulary."""
200+
201+ unknown_chars = sorted(set(text) - set(char_to_id))
202+ if unknown_chars:
203+ raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}")
204+
205+ return [char_to_id[char] for char in text]
206+
207+
208+def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str:
209+ """Convert token IDs back to text."""
210+
211+ return "".join(id_to_char[token_id] for token_id in token_ids.tolist())
212+
213+
214+def get_batch(
215+ data: torch.Tensor,
216+ batch_size: int,
217+ context_size: int,
218+ generator: torch.Generator,
219+) -> tuple[torch.Tensor, torch.Tensor]:
220+ """Return random fixed-context examples.
221+
222+ If the data is:
223+
224+ abcdef
225+
226+ and context_size is 4, one possible training example is:
227+
228+ input: abcd
229+ target: e
230+
231+ The target is one token, immediately after the context window.
232+ """
233+
234+ if len(data) <= context_size:
235+ raise ValueError(f"Need more than {context_size} tokens, got {len(data)}")
236+
237+ start_positions = torch.randint(
238+ high=len(data) - context_size,
239+ size=(batch_size,),
240+ generator=generator,
241+ )
242+
243+ inputs = torch.stack([data[start : start + context_size] for start in start_positions])
244+ targets = torch.stack([data[start + context_size] for start in start_positions])
245+
246+ return inputs, targets
247+
248+
249+@torch.no_grad()
250+def estimate_loss(
251+ model: FixedContextMLP,
252+ train_data: torch.Tensor,
253+ val_data: torch.Tensor,
254+ generator: torch.Generator,
255+) -> dict[str, float]:
256+ """Estimate train and validation loss without updating model weights."""
257+
258+ model.eval()
259+ losses: dict[str, float] = {}
260+
261+ for split_name, data in [("train", train_data), ("val", val_data)]:
262+ split_losses = torch.zeros(EVAL_ITERS)
263+
264+ for iteration in range(EVAL_ITERS):
265+ inputs, targets = get_batch(data, BATCH_SIZE, CONTEXT_SIZE, generator)
266+ _, loss = model(inputs, targets)
267+ assert loss is not None
268+ split_losses[iteration] = loss.item()
269+
270+ losses[split_name] = split_losses.mean().item()
271+
272+ model.train()
273+ return losses
274+
275+
276+def save_checkpoint(
277+ model: FixedContextMLP,
278+ vocab: dict[str, object],
279+ final_losses: dict[str, float],
280+) -> None:
281+ """Save model weights and enough metadata to reload the model later."""
282+
283+ CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
284+
285+ checkpoint = {
286+ "model_name": "fixed_context_mlp",
287+ "model_state_dict": model.state_dict(),
288+ "vocab": vocab,
289+ "hyperparameters": {
290+ "context_size": CONTEXT_SIZE,
291+ "embedding_size": EMBEDDING_SIZE,
292+ "hidden_size": HIDDEN_SIZE,
293+ "batch_size": BATCH_SIZE,
294+ "max_steps": MAX_STEPS,
295+ "learning_rate": LEARNING_RATE,
296+ "random_seed": RANDOM_SEED,
297+ "sample_temperature": SAMPLE_TEMPERATURE,
298+ },
299+ "final_losses": final_losses,
300+ }
301+
302+ torch.save(checkpoint, CHECKPOINT_PATH)
303+
304+
305+def main() -> None:
306+ torch.manual_seed(RANDOM_SEED)
307+ generator = torch.Generator().manual_seed(RANDOM_SEED)
308+
309+ train_data = load_token_ids(TRAIN_IDS_PATH)
310+ val_data = load_token_ids(VAL_IDS_PATH)
311+ vocab = load_vocab()
312+
313+ id_to_char = vocab["id_to_char"]
314+ char_to_id = vocab["char_to_id"]
315+ if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict):
316+ raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.")
317+
318+ if LEFT_PAD_CHAR not in char_to_id:
319+ raise ValueError(f"LEFT_PAD_CHAR={LEFT_PAD_CHAR!r} is not in the vocabulary")
320+
321+ vocab_size = len(id_to_char)
322+ model = FixedContextMLP(
323+ vocab_size=vocab_size,
324+ context_size=CONTEXT_SIZE,
325+ embedding_size=EMBEDDING_SIZE,
326+ hidden_size=HIDDEN_SIZE,
327+ )
328+ optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)
329+
330+ print("Training fixed-context MLP model")
331+ print(f" train tokens: {len(train_data)}")
332+ print(f" val tokens: {len(val_data)}")
333+ print(f" vocab size: {vocab_size}")
334+ print(f" context size: {CONTEXT_SIZE}")
335+ print(f" embedding size: {EMBEDDING_SIZE}")
336+ print(f" hidden size: {HIDDEN_SIZE}")
337+ print(f" parameters: {sum(parameter.numel() for parameter in model.parameters())}")
338+ print()
339+
340+ final_losses: dict[str, float] = {}
341+
342+ for step in range(MAX_STEPS + 1):
343+ if step % EVAL_INTERVAL == 0:
344+ losses = estimate_loss(model, train_data, val_data, generator)
345+ final_losses = losses
346+ print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
347+
348+ inputs, targets = get_batch(train_data, BATCH_SIZE, CONTEXT_SIZE, generator)
349+ _, loss = model(inputs, targets)
350+ assert loss is not None
351+
352+ optimizer.zero_grad(set_to_none=True)
353+ loss.backward()
354+ optimizer.step()
355+
356+ save_checkpoint(model, vocab, final_losses)
357+ print()
358+ print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}")
359+
360+ start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long)
361+ pad_token_id = char_to_id[LEFT_PAD_CHAR]
362+ generated_ids = model.generate(
363+ start_ids=start_ids,
364+ max_new_tokens=SAMPLE_NEW_TOKENS,
365+ pad_token_id=pad_token_id,
366+ temperature=SAMPLE_TEMPERATURE,
367+ generator=generator,
368+ )
369+ sample = decode(generated_ids[0], id_to_char)
370+
371+ print()
372+ print("Sample generation:")
373+ print("---")
374+ print(sample)
375+ print("---")
376+
377+
378+if __name__ == "__main__":
379+ main()