xonecas.io
768
La

learning-ai-lua-code-model / files

learning-ai-lua-code-model / src / lua_code_model / train_bigram.py

9.7 KB
1"""Train a tiny bigram language model with PyTorch.
2
3A language model learns to predict the next token.
4
5For this first model, the token is still just one character. The model is a
6"bigram" model because it only looks at one token of context:
7
8    current character -> predicted next character
9
10This is much simpler than GPT. GPT uses many previous tokens as context and a
11Transformer architecture. This file starts smaller so the PyTorch training loop
12is easy to study.
13
14Run from the project root after preparing the corpus:
15
16    PYTHONPATH=src uv run python -m lua_code_model.train_bigram
17"""
18
19from __future__ import annotations
20
21import json
22from pathlib import Path
23
24import torch
25import torch.nn as nn
26import torch.nn.functional as F
27
28
29PROJECT_ROOT = Path(__file__).resolve().parents[2]
30PROCESSED_DIR = PROJECT_ROOT / "data" / "processed"
31CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
32
33TRAIN_IDS_PATH = PROCESSED_DIR / "tiny_train_ids.txt"
34VAL_IDS_PATH = PROCESSED_DIR / "tiny_val_ids.txt"
35VOCAB_PATH = PROCESSED_DIR / "tiny_vocab.json"
36CHECKPOINT_PATH = CHECKPOINT_DIR / "bigram_tiny.pt"
37
38# These values are deliberately small so training is quick and inspectable.
39BATCH_SIZE = 32
40BLOCK_SIZE = 32
41MAX_STEPS = 1_000
42EVAL_INTERVAL = 100
43EVAL_ITERS = 20
44LEARNING_RATE = 0.1
45RANDOM_SEED = 1337
46SAMPLE_START_TEXT = "local"
47SAMPLE_NEW_TOKENS = 300
48
49
50class BigramLanguageModel(nn.Module):
51    """A tiny neural language model that predicts one character from one character.
52
53    The model owns one trainable table with shape:
54
55        vocab_size x vocab_size
56
57    For each input token ID, PyTorch selects one row from the table. That row is
58    a score for every possible next token.
59
60    Those raw scores are called logits. Cross-entropy loss turns the logits into
61    a training signal that says, "make the correct next token more likely."
62    """
63
64    def __init__(self, vocab_size: int) -> None:
65        super().__init__()
66        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)
67
68    def forward(
69        self,
70        input_ids: torch.Tensor,
71        targets: torch.Tensor | None = None,
72    ) -> tuple[torch.Tensor, torch.Tensor | None]:
73        """Return logits, and optionally the loss if targets are provided.
74
75        input_ids has shape:
76
77            batch_size x block_size
78
79        logits has shape:
80
81            batch_size x block_size x vocab_size
82
83        Each position in the input predicts the next token at the same position
84        in targets.
85        """
86
87        logits = self.token_embedding_table(input_ids)
88
89        if targets is None:
90            return logits, None
91
92        # PyTorch cross_entropy expects classes in the second dimension:
93        #   logits:  (examples, classes)
94        #   targets: (examples,)
95        #
96        # Our logits are 3D because we have a batch of short sequences. Flatten
97        # batch and time together so every character prediction becomes one
98        # training example.
99        batch_size, block_size, vocab_size = logits.shape
100        logits_flat = logits.view(batch_size * block_size, vocab_size)
101        targets_flat = targets.view(batch_size * block_size)
102        loss = F.cross_entropy(logits_flat, targets_flat)
103
104        return logits, loss
105
106    def generate(
107        self,
108        start_ids: torch.Tensor,
109        max_new_tokens: int,
110        generator: torch.Generator,
111    ) -> torch.Tensor:
112        """Sample new token IDs from the model.
113
114        Because this is a bigram model, only the most recent token matters when
115        choosing the next token.
116        """
117
118        generated = start_ids
119
120        for _ in range(max_new_tokens):
121            last_token = generated[:, -1:]
122            logits, _ = self(last_token)
123            next_token_logits = logits[:, -1, :]
124            probabilities = F.softmax(next_token_logits, dim=-1)
125            next_token = torch.multinomial(probabilities, num_samples=1, generator=generator)
126            generated = torch.cat((generated, next_token), dim=1)
127
128        return generated
129
130
131def load_token_ids(path: Path) -> torch.Tensor:
132    """Load one integer token ID per line into a PyTorch tensor."""
133
134    if not path.exists():
135        raise FileNotFoundError(
136            f"Missing {path}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
137        )
138
139    ids = [int(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
140    return torch.tensor(ids, dtype=torch.long)
141
142
143def load_vocab() -> dict[str, object]:
144    """Load the character vocabulary created by the preparation script."""
145
146    if not VOCAB_PATH.exists():
147        raise FileNotFoundError(
148            f"Missing {VOCAB_PATH}. Run: PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus"
149        )
150
151    return json.loads(VOCAB_PATH.read_text(encoding="utf-8"))
152
153
154def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
155    """Convert text to character token IDs using the saved vocabulary."""
156
157    unknown_chars = sorted(set(text) - set(char_to_id))
158    if unknown_chars:
159        raise ValueError(f"Text contains characters that are not in the tiny vocabulary: {unknown_chars}")
160
161    return [char_to_id[char] for char in text]
162
163
164def decode(token_ids: torch.Tensor, id_to_char: list[str]) -> str:
165    """Convert token IDs back to text."""
166
167    return "".join(id_to_char[token_id] for token_id in token_ids.tolist())
168
169
170def get_batch(
171    data: torch.Tensor,
172    batch_size: int,
173    block_size: int,
174    generator: torch.Generator,
175) -> tuple[torch.Tensor, torch.Tensor]:
176    """Return a random batch of input/target examples.
177
178    If the data is:
179
180        abcdef
181
182    and block_size is 4, one possible training example is:
183
184        input:  abcd
185        target: bcde
186
187    The target is shifted one character to the right because the task is next
188    character prediction.
189    """
190
191    if len(data) <= block_size:
192        raise ValueError(f"Need more than {block_size} tokens, got {len(data)}")
193
194    start_positions = torch.randint(
195        high=len(data) - block_size,
196        size=(batch_size,),
197        generator=generator,
198    )
199
200    inputs = torch.stack([data[start : start + block_size] for start in start_positions])
201    targets = torch.stack([data[start + 1 : start + block_size + 1] for start in start_positions])
202
203    return inputs, targets
204
205
206@torch.no_grad()
207def estimate_loss(
208    model: BigramLanguageModel,
209    train_data: torch.Tensor,
210    val_data: torch.Tensor,
211    generator: torch.Generator,
212) -> dict[str, float]:
213    """Estimate train and validation loss without updating model weights."""
214
215    model.eval()
216    losses: dict[str, float] = {}
217
218    for split_name, data in [("train", train_data), ("val", val_data)]:
219        split_losses = torch.zeros(EVAL_ITERS)
220
221        for iteration in range(EVAL_ITERS):
222            inputs, targets = get_batch(data, BATCH_SIZE, BLOCK_SIZE, generator)
223            _, loss = model(inputs, targets)
224            assert loss is not None
225            split_losses[iteration] = loss.item()
226
227        losses[split_name] = split_losses.mean().item()
228
229    model.train()
230    return losses
231
232
233def save_checkpoint(
234    model: BigramLanguageModel,
235    vocab: dict[str, object],
236    final_losses: dict[str, float],
237) -> None:
238    """Save model weights and enough metadata to reload the model later."""
239
240    CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
241
242    checkpoint = {
243        "model_name": "bigram",
244        "model_state_dict": model.state_dict(),
245        "vocab": vocab,
246        "hyperparameters": {
247            "batch_size": BATCH_SIZE,
248            "block_size": BLOCK_SIZE,
249            "max_steps": MAX_STEPS,
250            "learning_rate": LEARNING_RATE,
251            "random_seed": RANDOM_SEED,
252        },
253        "final_losses": final_losses,
254    }
255
256    torch.save(checkpoint, CHECKPOINT_PATH)
257
258
259def main() -> None:
260    torch.manual_seed(RANDOM_SEED)
261    generator = torch.Generator().manual_seed(RANDOM_SEED)
262
263    train_data = load_token_ids(TRAIN_IDS_PATH)
264    val_data = load_token_ids(VAL_IDS_PATH)
265    vocab = load_vocab()
266
267    id_to_char = vocab["id_to_char"]
268    char_to_id = vocab["char_to_id"]
269    if not isinstance(id_to_char, list) or not isinstance(char_to_id, dict):
270        raise TypeError("Unexpected vocabulary format. Re-run prepare_tiny_corpus.py.")
271
272    vocab_size = len(id_to_char)
273    model = BigramLanguageModel(vocab_size)
274    optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)
275
276    print("Training tiny bigram model")
277    print(f"  train tokens: {len(train_data)}")
278    print(f"  val tokens:   {len(val_data)}")
279    print(f"  vocab size:   {vocab_size}")
280    print(f"  parameters:   {sum(parameter.numel() for parameter in model.parameters())}")
281    print()
282
283    final_losses: dict[str, float] = {}
284
285    for step in range(MAX_STEPS + 1):
286        if step % EVAL_INTERVAL == 0:
287            losses = estimate_loss(model, train_data, val_data, generator)
288            final_losses = losses
289            print(f"step {step:4d}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
290
291        inputs, targets = get_batch(train_data, BATCH_SIZE, BLOCK_SIZE, generator)
292        _, loss = model(inputs, targets)
293        assert loss is not None
294
295        # Standard PyTorch training pattern:
296        # 1. clear old gradients
297        # 2. compute new gradients with backpropagation
298        # 3. update parameters with the optimizer
299        optimizer.zero_grad(set_to_none=True)
300        loss.backward()
301        optimizer.step()
302
303    save_checkpoint(model, vocab, final_losses)
304    print()
305    print(f"Saved checkpoint: {CHECKPOINT_PATH.relative_to(PROJECT_ROOT)}")
306
307    start_ids = torch.tensor([encode(SAMPLE_START_TEXT, char_to_id)], dtype=torch.long)
308    generated_ids = model.generate(start_ids, SAMPLE_NEW_TOKENS, generator)
309    sample = decode(generated_ids[0], id_to_char)
310
311    print()
312    print("Sample generation:")
313    print("---")
314    print(sample)
315    print("---")
316
317
318if __name__ == "__main__":
319    main()