xonecas.io
768
La

learning-ai-lua-code-model / commit 88614c7

feat: first dataset

xonecas2026-06-14 88614c76b0cf97f42db2025bd3ac69750cd9ac9d parent 6852eee

10 files changed

.gitignore+10−0
@@ -0,0 +1,10 @@
1+# Python
2+__pycache__/
3+*.py[cod]
4+.venv/
5+
6+# Generated local data and training outputs
7+data/processed/*
8+!data/processed/.gitkeep
9+checkpoints/
10+runs/
README.md+41−0
@@ -0,0 +1,41 @@
1+# Lua Code Model
2+
3+A tiny, educational language-model project focused on Lua code and Lua documentation.
4+
5+The goal is to learn the pieces of a GPT-style model one at a time, starting with the smallest possible end-to-end dataset pipeline.
6+
7+## Current milestone
8+
9+This repo currently contains:
10+
11+- a small hand-written Lua corpus in `data/raw/tiny_lua_corpus.txt`
12+- a source log in `data/SOURCES.md`
13+- a standard-library-only preparation script in `src/lua_code_model/prepare_tiny_corpus.py`
14+- short learning notes in `notes/001-first-dataset.md`
15+
16+## Prepare the tiny corpus
17+
18+From the project root:
19+
20+```bash
21+PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus
22+```
23+
24+This project currently has no third-party Python dependencies. If you are not using `uv`, the same script can be run with plain Python:
25+
26+```bash
27+PYTHONPATH=src python -m lua_code_model.prepare_tiny_corpus
28+```
29+
30+The script writes generated files to `data/processed/`:
31+
32+- `tiny_train.txt` and `tiny_val.txt`: human-readable text splits
33+- `tiny_train_ids.txt` and `tiny_val_ids.txt`: one character token ID per line
34+- `tiny_vocab.json`: character-to-ID mapping
35+- `tiny_metadata.json`: basic dataset statistics
36+
37+## Next likely step
38+
39+After this pipeline is clear, the next milestone is to add PyTorch and train a very small model to predict the next character.
40+
41+We should ask before adding PyTorch because it is the first third-party dependency.
data/SOURCES.md+11−0
@@ -0,0 +1,11 @@
1+# Data sources
2+
3+Every dataset added to this project should be recorded here before training on it.
4+
5+## `raw/tiny_lua_corpus.txt`
6+
7+- Source: Handwritten for this educational repository.
8+- License: No third-party source text or code was copied. Treat it as project-owned example data.
9+- Included: Small Lua snippets showing comments, tables, functions, loops, string concatenation, conditionals, and colon-method syntax.
10+- Excluded: External code, generated code, vendored code, minified code, and obfuscated code.
11+- Cleaning/filtering: None needed; the file was written directly as a tiny first corpus.
data/processed/.gitkeep+0−0
data/raw/tiny_lua_corpus.txt+47−0
@@ -0,0 +1,47 @@
1+-- Tiny Lua corpus for first language model experiments.
2+-- Source: handwritten for this educational repository.
3+-- It is intentionally small so the first model can overfit it quickly.
4+
5+-- In Lua, tables are the main data structure.
6+local inventory = {
7+ apples = 3,
8+ oranges = 5,
9+ potions = 2,
10+}
11+
12+local function count_items(items)
13+ local total = 0
14+
15+ for name, amount in pairs(items) do
16+ print(name .. ": " .. amount)
17+ total = total + amount
18+ end
19+
20+ return total
21+end
22+
23+local total = count_items(inventory)
24+
25+if total > 5 then
26+ print("enough supplies")
27+else
28+ print("pack more supplies")
29+end
30+
31+-- A tiny object-like table.
32+-- The colon syntax passes self as the first argument.
33+local player = {}
34+player.__index = player
35+
36+function player.new(name)
37+ return setmetatable({ name = name, health = 10 }, player)
38+end
39+
40+function player:heal(amount)
41+ self.health = self.health + amount
42+ return self.health
43+end
44+
45+local ada = player.new("Ada")
46+ada:heal(4)
47+print(ada.name .. " has " .. ada.health .. " health")
notes/001-first-dataset.md+34−0
@@ -0,0 +1,34 @@
1+# First dataset pipeline
2+
3+A language model cannot read text directly. First, we turn text into numbers.
4+
5+For this first milestone, we use a **character-level tokenizer**:
6+
7+- each unique character in the corpus gets an integer ID
8+- encoding means `text -> list[int]`
9+- decoding means `list[int] -> text`
10+
11+Example idea:
12+
13+```text
14+"lua" -> [12, 20, 3]
15+```
16+
17+The exact numbers do not matter. What matters is that the same vocabulary is used consistently.
18+
19+## Why split into train and validation?
20+
21+The model learns from the **training** split.
22+
23+The **validation** split is held back so we can ask, "Does the model only memorize the training text, or is it learning patterns that also help on text it did not train on?"
24+
25+For the tiny first corpus, overfitting is expected and useful. It proves that the data path, model, loss, optimizer, checkpointing, and sampling all work before we scale up.
26+
27+## Files produced by the preparation script
28+
29+Running `prepare_tiny_corpus.py` creates:
30+
31+- `tiny_train.txt` / `tiny_val.txt`: readable text splits
32+- `tiny_train_ids.txt` / `tiny_val_ids.txt`: token IDs for model training
33+- `tiny_vocab.json`: the character vocabulary
34+- `tiny_metadata.json`: counts and paths for quick inspection
pyproject.toml+10−0
@@ -0,0 +1,10 @@
1+[project]
2+name = "lua-code-model"
3+version = "0.1.0"
4+description = "A tiny educational GPT-style model for Lua code experiments."
5+readme = "README.md"
6+requires-python = ">=3.11"
7+dependencies = []
8+
9+[tool.uv]
10+package = false
src/lua_code_model/__init__.py+3−0
@@ -0,0 +1,3 @@
1+"""Tiny educational language-model experiments for Lua code."""
2+
3+__all__ = []
src/lua_code_model/prepare_tiny_corpus.py+155−0
@@ -0,0 +1,155 @@
1+"""Prepare the first tiny Lua corpus.
2+
3+This script is intentionally simple and uses only the Python standard library.
4+It teaches the first data-processing ideas we need before training a model:
5+
6+1. read raw text
7+2. build a character-level vocabulary
8+3. encode text characters as integer token IDs
9+4. split token IDs into train and validation sets
10+5. write files that later training code can read
11+
12+Run from the project root:
13+
14+ PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus
15+
16+Or, without uv:
17+
18+ PYTHONPATH=src python -m lua_code_model.prepare_tiny_corpus
19+"""
20+
21+from __future__ import annotations
22+
23+import json
24+from pathlib import Path
25+from typing import Iterable
26+
27+
28+PROJECT_ROOT = Path(__file__).resolve().parents[2]
29+RAW_CORPUS_PATH = PROJECT_ROOT / "data" / "raw" / "tiny_lua_corpus.txt"
30+OUTPUT_DIR = PROJECT_ROOT / "data" / "processed"
31+
32+TRAIN_TEXT_PATH = OUTPUT_DIR / "tiny_train.txt"
33+VAL_TEXT_PATH = OUTPUT_DIR / "tiny_val.txt"
34+TRAIN_IDS_PATH = OUTPUT_DIR / "tiny_train_ids.txt"
35+VAL_IDS_PATH = OUTPUT_DIR / "tiny_val_ids.txt"
36+VOCAB_PATH = OUTPUT_DIR / "tiny_vocab.json"
37+METADATA_PATH = OUTPUT_DIR / "tiny_metadata.json"
38+
39+TRAIN_FRACTION = 0.90
40+
41+
42+def build_vocab(text: str) -> tuple[dict[str, int], list[str]]:
43+ """Create character <-> integer mappings from the corpus text.
44+
45+ A tokenizer converts text into tokens. For this first version, a token is
46+ just one character. Each unique character receives a stable integer ID.
47+ """
48+
49+ id_to_char = sorted(set(text))
50+ char_to_id = {char: token_id for token_id, char in enumerate(id_to_char)}
51+ return char_to_id, id_to_char
52+
53+
54+def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
55+ """Convert text into integer token IDs."""
56+
57+ return [char_to_id[char] for char in text]
58+
59+
60+def decode(token_ids: Iterable[int], id_to_char: list[str]) -> str:
61+ """Convert integer token IDs back into text."""
62+
63+ return "".join(id_to_char[token_id] for token_id in token_ids)
64+
65+
66+def split_train_val(token_ids: list[int], train_fraction: float) -> tuple[list[int], list[int]]:
67+ """Split token IDs into training and validation portions.
68+
69+ The training split is what the model learns from. The validation split is
70+ held back so we can measure whether the model is only memorizing.
71+ """
72+
73+ if len(token_ids) < 2:
74+ raise ValueError("Need at least two tokens to create train and validation splits.")
75+
76+ split_at = int(len(token_ids) * train_fraction)
77+
78+ # Keep both splits non-empty, even for tiny corpora.
79+ split_at = max(1, min(len(token_ids) - 1, split_at))
80+
81+ return token_ids[:split_at], token_ids[split_at:]
82+
83+
84+def write_token_ids(path: Path, token_ids: Iterable[int]) -> None:
85+ """Write one token ID per line so the file is easy to inspect."""
86+
87+ path.write_text("\n".join(str(token_id) for token_id in token_ids) + "\n", encoding="utf-8")
88+
89+
90+def relative(path: Path) -> str:
91+ """Format paths relative to the project root for readable metadata."""
92+
93+ return str(path.relative_to(PROJECT_ROOT))
94+
95+
96+def main() -> None:
97+ if not RAW_CORPUS_PATH.exists():
98+ raise FileNotFoundError(f"Missing raw corpus: {RAW_CORPUS_PATH}")
99+
100+ text = RAW_CORPUS_PATH.read_text(encoding="utf-8")
101+ if not text.strip():
102+ raise ValueError(f"Raw corpus is empty: {RAW_CORPUS_PATH}")
103+
104+ char_to_id, id_to_char = build_vocab(text)
105+ token_ids = encode(text, char_to_id)
106+
107+ # Round-trip check: encoding and then decoding should recover the exact text.
108+ if decode(token_ids, id_to_char) != text:
109+ raise RuntimeError("Tokenizer round-trip failed: decoded text differs from input text.")
110+
111+ train_ids, val_ids = split_train_val(token_ids, TRAIN_FRACTION)
112+ train_text = decode(train_ids, id_to_char)
113+ val_text = decode(val_ids, id_to_char)
114+
115+ OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
116+
117+ TRAIN_TEXT_PATH.write_text(train_text, encoding="utf-8")
118+ VAL_TEXT_PATH.write_text(val_text, encoding="utf-8")
119+ write_token_ids(TRAIN_IDS_PATH, train_ids)
120+ write_token_ids(VAL_IDS_PATH, val_ids)
121+
122+ vocab = {
123+ "tokenizer": "character",
124+ "vocab_size": len(id_to_char),
125+ "id_to_char": id_to_char,
126+ "char_to_id": char_to_id,
127+ }
128+ VOCAB_PATH.write_text(json.dumps(vocab, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
129+
130+ metadata = {
131+ "raw_corpus_path": relative(RAW_CORPUS_PATH),
132+ "train_text_path": relative(TRAIN_TEXT_PATH),
133+ "validation_text_path": relative(VAL_TEXT_PATH),
134+ "train_ids_path": relative(TRAIN_IDS_PATH),
135+ "validation_ids_path": relative(VAL_IDS_PATH),
136+ "vocab_path": relative(VOCAB_PATH),
137+ "tokenizer": "character",
138+ "num_characters": len(text),
139+ "vocab_size": len(id_to_char),
140+ "train_tokens": len(train_ids),
141+ "validation_tokens": len(val_ids),
142+ "train_fraction_requested": TRAIN_FRACTION,
143+ }
144+ METADATA_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
145+
146+ print("Prepared tiny Lua corpus")
147+ print(f" raw characters: {len(text)}")
148+ print(f" vocab size: {len(id_to_char)}")
149+ print(f" train tokens: {len(train_ids)}")
150+ print(f" validation tokens: {len(val_ids)}")
151+ print(f" output directory: {relative(OUTPUT_DIR)}")
152+
153+
154+if __name__ == "__main__":
155+ main()
uv.lock+8−0
@@ -0,0 +1,8 @@
1+version = 1
2+revision = 3
3+requires-python = ">=3.11"
4+
5+[[package]]
6+name = "lua-code-model"
7+version = "0.1.0"
8+source = { virtual = "." }