@@ -0,0 +1,155 @@
1+"""Prepare the first tiny Lua corpus.
2+
3+This script is intentionally simple and uses only the Python standard library.
4+It teaches the first data-processing ideas we need before training a model:
5+
6+1. read raw text
7+2. build a character-level vocabulary
8+3. encode text characters as integer token IDs
9+4. split token IDs into train and validation sets
10+5. write files that later training code can read
11+
12+Run from the project root:
13+
14+ PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus
15+
16+Or, without uv:
17+
18+ PYTHONPATH=src python -m lua_code_model.prepare_tiny_corpus
19+"""
20+
21+from __future__ import annotations
22+
23+import json
24+from pathlib import Path
25+from typing import Iterable
26+
27+
28+PROJECT_ROOT = Path(__file__).resolve().parents[2]
29+RAW_CORPUS_PATH = PROJECT_ROOT / "data" / "raw" / "tiny_lua_corpus.txt"
30+OUTPUT_DIR = PROJECT_ROOT / "data" / "processed"
31+
32+TRAIN_TEXT_PATH = OUTPUT_DIR / "tiny_train.txt"
33+VAL_TEXT_PATH = OUTPUT_DIR / "tiny_val.txt"
34+TRAIN_IDS_PATH = OUTPUT_DIR / "tiny_train_ids.txt"
35+VAL_IDS_PATH = OUTPUT_DIR / "tiny_val_ids.txt"
36+VOCAB_PATH = OUTPUT_DIR / "tiny_vocab.json"
37+METADATA_PATH = OUTPUT_DIR / "tiny_metadata.json"
38+
39+TRAIN_FRACTION = 0.90
40+
41+
42+def build_vocab(text: str) -> tuple[dict[str, int], list[str]]:
43+ """Create character <-> integer mappings from the corpus text.
44+
45+ A tokenizer converts text into tokens. For this first version, a token is
46+ just one character. Each unique character receives a stable integer ID.
47+ """
48+
49+ id_to_char = sorted(set(text))
50+ char_to_id = {char: token_id for token_id, char in enumerate(id_to_char)}
51+ return char_to_id, id_to_char
52+
53+
54+def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
55+ """Convert text into integer token IDs."""
56+
57+ return [char_to_id[char] for char in text]
58+
59+
60+def decode(token_ids: Iterable[int], id_to_char: list[str]) -> str:
61+ """Convert integer token IDs back into text."""
62+
63+ return "".join(id_to_char[token_id] for token_id in token_ids)
64+
65+
66+def split_train_val(token_ids: list[int], train_fraction: float) -> tuple[list[int], list[int]]:
67+ """Split token IDs into training and validation portions.
68+
69+ The training split is what the model learns from. The validation split is
70+ held back so we can measure whether the model is only memorizing.
71+ """
72+
73+ if len(token_ids) < 2:
74+ raise ValueError("Need at least two tokens to create train and validation splits.")
75+
76+ split_at = int(len(token_ids) * train_fraction)
77+
78+ # Keep both splits non-empty, even for tiny corpora.
79+ split_at = max(1, min(len(token_ids) - 1, split_at))
80+
81+ return token_ids[:split_at], token_ids[split_at:]
82+
83+
84+def write_token_ids(path: Path, token_ids: Iterable[int]) -> None:
85+ """Write one token ID per line so the file is easy to inspect."""
86+
87+ path.write_text("\n".join(str(token_id) for token_id in token_ids) + "\n", encoding="utf-8")
88+
89+
90+def relative(path: Path) -> str:
91+ """Format paths relative to the project root for readable metadata."""
92+
93+ return str(path.relative_to(PROJECT_ROOT))
94+
95+
96+def main() -> None:
97+ if not RAW_CORPUS_PATH.exists():
98+ raise FileNotFoundError(f"Missing raw corpus: {RAW_CORPUS_PATH}")
99+
100+ text = RAW_CORPUS_PATH.read_text(encoding="utf-8")
101+ if not text.strip():
102+ raise ValueError(f"Raw corpus is empty: {RAW_CORPUS_PATH}")
103+
104+ char_to_id, id_to_char = build_vocab(text)
105+ token_ids = encode(text, char_to_id)
106+
107+ # Round-trip check: encoding and then decoding should recover the exact text.
108+ if decode(token_ids, id_to_char) != text:
109+ raise RuntimeError("Tokenizer round-trip failed: decoded text differs from input text.")
110+
111+ train_ids, val_ids = split_train_val(token_ids, TRAIN_FRACTION)
112+ train_text = decode(train_ids, id_to_char)
113+ val_text = decode(val_ids, id_to_char)
114+
115+ OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
116+
117+ TRAIN_TEXT_PATH.write_text(train_text, encoding="utf-8")
118+ VAL_TEXT_PATH.write_text(val_text, encoding="utf-8")
119+ write_token_ids(TRAIN_IDS_PATH, train_ids)
120+ write_token_ids(VAL_IDS_PATH, val_ids)
121+
122+ vocab = {
123+ "tokenizer": "character",
124+ "vocab_size": len(id_to_char),
125+ "id_to_char": id_to_char,
126+ "char_to_id": char_to_id,
127+ }
128+ VOCAB_PATH.write_text(json.dumps(vocab, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
129+
130+ metadata = {
131+ "raw_corpus_path": relative(RAW_CORPUS_PATH),
132+ "train_text_path": relative(TRAIN_TEXT_PATH),
133+ "validation_text_path": relative(VAL_TEXT_PATH),
134+ "train_ids_path": relative(TRAIN_IDS_PATH),
135+ "validation_ids_path": relative(VAL_IDS_PATH),
136+ "vocab_path": relative(VOCAB_PATH),
137+ "tokenizer": "character",
138+ "num_characters": len(text),
139+ "vocab_size": len(id_to_char),
140+ "train_tokens": len(train_ids),
141+ "validation_tokens": len(val_ids),
142+ "train_fraction_requested": TRAIN_FRACTION,
143+ }
144+ METADATA_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
145+
146+ print("Prepared tiny Lua corpus")
147+ print(f" raw characters: {len(text)}")
148+ print(f" vocab size: {len(id_to_char)}")
149+ print(f" train tokens: {len(train_ids)}")
150+ print(f" validation tokens: {len(val_ids)}")
151+ print(f" output directory: {relative(OUTPUT_DIR)}")
152+
153+
154+if __name__ == "__main__":
155+ main()