xonecas.io
768
La

learning-ai-lua-code-model / files

learning-ai-lua-code-model / src / lua_code_model / prepare_tiny_corpus.py

5.2 KB
1"""Prepare the first tiny Lua corpus.
2
3This script is intentionally simple and uses only the Python standard library.
4It teaches the first data-processing ideas we need before training a model:
5
61. read raw text
72. build a character-level vocabulary
83. encode text characters as integer token IDs
94. split token IDs into train and validation sets
105. write files that later training code can read
11
12Run from the project root:
13
14    PYTHONPATH=src uv run python -m lua_code_model.prepare_tiny_corpus
15
16Or, without uv:
17
18    PYTHONPATH=src python -m lua_code_model.prepare_tiny_corpus
19"""
20
21from __future__ import annotations
22
23import json
24from pathlib import Path
25from typing import Iterable
26
27
28PROJECT_ROOT = Path(__file__).resolve().parents[2]
29RAW_CORPUS_PATH = PROJECT_ROOT / "data" / "raw" / "tiny_lua_corpus.txt"
30OUTPUT_DIR = PROJECT_ROOT / "data" / "processed"
31
32TRAIN_TEXT_PATH = OUTPUT_DIR / "tiny_train.txt"
33VAL_TEXT_PATH = OUTPUT_DIR / "tiny_val.txt"
34TRAIN_IDS_PATH = OUTPUT_DIR / "tiny_train_ids.txt"
35VAL_IDS_PATH = OUTPUT_DIR / "tiny_val_ids.txt"
36VOCAB_PATH = OUTPUT_DIR / "tiny_vocab.json"
37METADATA_PATH = OUTPUT_DIR / "tiny_metadata.json"
38
39TRAIN_FRACTION = 0.90
40
41
42def build_vocab(text: str) -> tuple[dict[str, int], list[str]]:
43    """Create character <-> integer mappings from the corpus text.
44
45    A tokenizer converts text into tokens. For this first version, a token is
46    just one character. Each unique character receives a stable integer ID.
47    """
48
49    id_to_char = sorted(set(text))
50    char_to_id = {char: token_id for token_id, char in enumerate(id_to_char)}
51    return char_to_id, id_to_char
52
53
54def encode(text: str, char_to_id: dict[str, int]) -> list[int]:
55    """Convert text into integer token IDs."""
56
57    return [char_to_id[char] for char in text]
58
59
60def decode(token_ids: Iterable[int], id_to_char: list[str]) -> str:
61    """Convert integer token IDs back into text."""
62
63    return "".join(id_to_char[token_id] for token_id in token_ids)
64
65
66def split_train_val(token_ids: list[int], train_fraction: float) -> tuple[list[int], list[int]]:
67    """Split token IDs into training and validation portions.
68
69    The training split is what the model learns from. The validation split is
70    held back so we can measure whether the model is only memorizing.
71    """
72
73    if len(token_ids) < 2:
74        raise ValueError("Need at least two tokens to create train and validation splits.")
75
76    split_at = int(len(token_ids) * train_fraction)
77
78    # Keep both splits non-empty, even for tiny corpora.
79    split_at = max(1, min(len(token_ids) - 1, split_at))
80
81    return token_ids[:split_at], token_ids[split_at:]
82
83
84def write_token_ids(path: Path, token_ids: Iterable[int]) -> None:
85    """Write one token ID per line so the file is easy to inspect."""
86
87    path.write_text("\n".join(str(token_id) for token_id in token_ids) + "\n", encoding="utf-8")
88
89
90def relative(path: Path) -> str:
91    """Format paths relative to the project root for readable metadata."""
92
93    return str(path.relative_to(PROJECT_ROOT))
94
95
96def main() -> None:
97    if not RAW_CORPUS_PATH.exists():
98        raise FileNotFoundError(f"Missing raw corpus: {RAW_CORPUS_PATH}")
99
100    text = RAW_CORPUS_PATH.read_text(encoding="utf-8")
101    if not text.strip():
102        raise ValueError(f"Raw corpus is empty: {RAW_CORPUS_PATH}")
103
104    char_to_id, id_to_char = build_vocab(text)
105    token_ids = encode(text, char_to_id)
106
107    # Round-trip check: encoding and then decoding should recover the exact text.
108    if decode(token_ids, id_to_char) != text:
109        raise RuntimeError("Tokenizer round-trip failed: decoded text differs from input text.")
110
111    train_ids, val_ids = split_train_val(token_ids, TRAIN_FRACTION)
112    train_text = decode(train_ids, id_to_char)
113    val_text = decode(val_ids, id_to_char)
114
115    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
116
117    TRAIN_TEXT_PATH.write_text(train_text, encoding="utf-8")
118    VAL_TEXT_PATH.write_text(val_text, encoding="utf-8")
119    write_token_ids(TRAIN_IDS_PATH, train_ids)
120    write_token_ids(VAL_IDS_PATH, val_ids)
121
122    vocab = {
123        "tokenizer": "character",
124        "vocab_size": len(id_to_char),
125        "id_to_char": id_to_char,
126        "char_to_id": char_to_id,
127    }
128    VOCAB_PATH.write_text(json.dumps(vocab, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
129
130    metadata = {
131        "raw_corpus_path": relative(RAW_CORPUS_PATH),
132        "train_text_path": relative(TRAIN_TEXT_PATH),
133        "validation_text_path": relative(VAL_TEXT_PATH),
134        "train_ids_path": relative(TRAIN_IDS_PATH),
135        "validation_ids_path": relative(VAL_IDS_PATH),
136        "vocab_path": relative(VOCAB_PATH),
137        "tokenizer": "character",
138        "num_characters": len(text),
139        "vocab_size": len(id_to_char),
140        "train_tokens": len(train_ids),
141        "validation_tokens": len(val_ids),
142        "train_fraction_requested": TRAIN_FRACTION,
143    }
144    METADATA_PATH.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
145
146    print("Prepared tiny Lua corpus")
147    print(f"  raw characters:    {len(text)}")
148    print(f"  vocab size:        {len(id_to_char)}")
149    print(f"  train tokens:      {len(train_ids)}")
150    print(f"  validation tokens: {len(val_ids)}")
151    print(f"  output directory:  {relative(OUTPUT_DIR)}")
152
153
154if __name__ == "__main__":
155    main()