768
La
learning-ai-lua-code-model / files
learning-ai-lua-code-model / src / lua_code_model / prepare_tiny_corpus.py
5.2 KB
1"""Prepare the first tiny Lua corpus. 2 3This script is intentionally simple and uses only the Python standard library. 4It teaches the first data-processing ideas we need before training a model: 5 61. read raw text 72. build a character-level vocabulary 83. encode text characters as integer token IDs 94. split token IDs into train and validation sets 105. write files that later training code can read 11 12Run from the project root: 13 14 PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus 15 16Or, without uv: 17 18 PYTHONPATH=src python -m lua_code_model.prepare_tiny_corpus 19""" 20 21from __future__ import annotations 22 23import json 24from pathlib import Path 25from typing import Iterable 26 27 28PROJECT_ROOT = Path(__file__).resolve().parents[2] 29RAW_CORPUS_PATH = PROJECT_ROOT / "data" / "raw" / "tiny_lua_corpus.txt" 30OUTPUT_DIR = PROJECT_ROOT / "data" / "processed" 31 32TRAIN_TEXT_PATH = OUTPUT_DIR / "tiny_train.txt" 33VAL_TEXT_PATH = OUTPUT_DIR / "tiny_val.txt" 34TRAIN_IDS_PATH = OUTPUT_DIR / "tiny_train_ids.txt" 35VAL_IDS_PATH = OUTPUT_DIR / "tiny_val_ids.txt" 36VOCAB_PATH = OUTPUT_DIR / "tiny_vocab.json" 37METADATA_PATH = OUTPUT_DIR / "tiny_metadata.json" 38 39TRAIN_FRACTION = 0.90 40 41 42def build_vocab(text: str) -> tuple[dict[str, int], list[str]]: 43 """Create character <-> integer mappings from the corpus text. 44 45 A tokenizer converts text into tokens. For this first version, a token is 46 just one character. Each unique character receives a stable integer ID. 47 """ 48 49 id_to_char = sorted(set(text)) 50 char_to_id = {char: token_id for token_id, char in enumerate(id_to_char)} 51 return char_to_id, id_to_char 52 53 54def encode(text: str, char_to_id: dict[str, int]) -> list[int]: 55 """Convert text into integer token IDs.""" 56 57 return [char_to_id[char] for char in text] 58 59 60def decode(token_ids: Iterable[int], id_to_char: list[str]) -> str: 61 """Convert integer token IDs back into text.""" 62 63 return "".join(id_to_char[token_id] for token_id in token_ids) 64 65 66def split_train_val(token_ids: list[int], train_fraction: float) -> tuple[list[int], list[int]]: 67 """Split token IDs into training and validation portions. 68 69 The training split is what the model learns from. The validation split is 70 held back so we can measure whether the model is only memorizing. 71 """ 72 73 if len(token_ids) < 2: 74 raise ValueError("Need at least two tokens to create train and validation splits.") 75 76 split_at = int(len(token_ids) * train_fraction) 77 78 # Keep both splits non-empty, even for tiny corpora. 79 split_at = max(1, min(len(token_ids) - 1, split_at)) 80 81 return token_ids[:split_at], token_ids[split_at:] 82 83 84def write_token_ids(path: Path, token_ids: Iterable[int]) -> None: 85 """Write one token ID per line so the file is easy to inspect.""" 86 87 path.write_text("\n".join(str(token_id) for token_id in token_ids) + "\n", encoding="utf-8") 88 89 90def relative(path: Path) -> str: 91 """Format paths relative to the project root for readable metadata.""" 92 93 return str(path.relative_to(PROJECT_ROOT)) 94 95 96def main() -> None: 97 if not RAW_CORPUS_PATH.exists(): 98 raise FileNotFoundError(f"Missing raw corpus: {RAW_CORPUS_PATH}") 99 100 text = RAW_CORPUS_PATH.read_text(encoding="utf-8") 101 if not text.strip(): 102 raise ValueError(f"Raw corpus is empty: {RAW_CORPUS_PATH}") 103 104 char_to_id, id_to_char = build_vocab(text) 105 token_ids = encode(text, char_to_id) 106 107 # Round-trip check: encoding and then decoding should recover the exact text. 108 if decode(token_ids, id_to_char) != text: 109 raise RuntimeError("Tokenizer round-trip failed: decoded text differs from input text.") 110 111 train_ids, val_ids = split_train_val(token_ids, TRAIN_FRACTION) 112 train_text = decode(train_ids, id_to_char) 113 val_text = decode(val_ids, id_to_char) 114 115 OUTPUT_DIR.mkdir(parents=True, exist_ok=True) 116 117 TRAIN_TEXT_PATH.write_text(train_text, encoding="utf-8") 118 VAL_TEXT_PATH.write_text(val_text, encoding="utf-8") 119 write_token_ids(TRAIN_IDS_PATH, train_ids) 120 write_token_ids(VAL_IDS_PATH, val_ids) 121 122 vocab = { 123 "tokenizer": "character", 124 "vocab_size": len(id_to_char), 125 "id_to_char": id_to_char, 126 "char_to_id": char_to_id, 127 } 128 VOCAB_PATH.write_text(json.dumps(vocab, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") 129 130 metadata = { 131 "raw_corpus_path": relative(RAW_CORPUS_PATH), 132 "train_text_path": relative(TRAIN_TEXT_PATH), 133 "validation_text_path": relative(VAL_TEXT_PATH), 134 "train_ids_path": relative(TRAIN_IDS_PATH), 135 "validation_ids_path": relative(VAL_IDS_PATH), 136 "vocab_path": relative(VOCAB_PATH), 137 "tokenizer": "character", 138 "num_characters": len(text), 139 "vocab_size": len(id_to_char), 140 "train_tokens": len(train_ids), 141 "validation_tokens": len(val_ids), 142 "train_fraction_requested": TRAIN_FRACTION, 143 } 144 METADATA_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") 145 146 print("Prepared tiny Lua corpus") 147 print(f" raw characters: {len(text)}") 148 print(f" vocab size: {len(id_to_char)}") 149 print(f" train tokens: {len(train_ids)}") 150 print(f" validation tokens: {len(val_ids)}") 151 print(f" output directory: {relative(OUTPUT_DIR)}") 152 153 154if __name__ == "__main__": 155 main()