xonecas.io
79113
Ma

mini-coding-agent / files

mini-coding-agent / tools / gen_width.py

5.3 KB
1#!/usr/bin/env python3
2"""Generates src/tui/width_tables.zig from the Unicode Character Database.
3
4Every table is a sorted list of non-overlapping inclusive codepoint ranges, so
5lookup is a binary search and a normal `zig build` needs no network. Run
6`python3 tools/gen_width.py` to refresh against a newer UCD; the version it
7reads is printed and recorded in the file header.
8"""
9
10import sys
11import urllib.request
12
13BASE = "https://www.unicode.org/Public/UCD/latest/ucd/"
14
15FILES = {
16    "eaw": "EastAsianWidth.txt",
17    "gc": "extracted/DerivedGeneralCategory.txt",
18    "dcp": "DerivedCoreProperties.txt",
19    "emoji": "emoji/emoji-data.txt",
20    "gbp": "auxiliary/GraphemeBreakProperty.txt",
21}
22
23
24def fetch(name):
25    url = BASE + FILES[name]
26    try:
27        with urllib.request.urlopen(url) as r:
28            return r.read().decode("utf-8")
29    except Exception as e:  # pragma: no cover
30        sys.exit(f"gen_width: cannot fetch {url}: {e}")
31
32
33def parse(text, field=1):
34    """Yields (lo, hi, property) for every non-comment line of a UCD file.
35
36    A line may carry several semicolon fields; `field` picks one, and -1 picks
37    the last. The InCB lines of DerivedCoreProperties are `range; InCB; Value`,
38    so they are read with field=-1.
39    """
40    for line in text.splitlines():
41        line = line.split("#", 1)[0].strip()
42        if not line or ";" not in line:
43            continue
44        parts = [p.strip() for p in line.split(";")]
45        rng = parts[0]
46        lo, _, hi = rng.partition("..")
47        yield int(lo, 16), int(hi or lo, 16), parts[field]
48
49
50def merge(rows):
51    """Sorts and coalesces (lo, hi) pairs into disjoint ranges."""
52    rows = sorted(set(rows))
53    out = []
54    for lo, hi in rows:
55        if out and lo <= out[-1][1] + 1:
56            out[-1][1] = max(out[-1][1], hi)
57        else:
58            out.append([lo, hi])
59    return [(lo, hi) for lo, hi in out]
60
61
62def select(rows, wanted):
63    return merge([(lo, hi) for lo, hi, prop in rows if prop in wanted])
64
65
66def classes(rows, wanted):
67    """Merges per-property ranges into a distinct-class range list."""
68    by_prop = {}
69    for lo, hi, prop in rows:
70        if prop in wanted:
71            by_prop.setdefault(prop, []).append((lo, hi))
72    out = []
73    for prop, ranges in by_prop.items():
74        for lo, hi in merge(ranges):
75            out.append((lo, hi, prop))
76    out.sort()
77    return out
78
79
80def emit_range(name, ranges):
81    body = ",\n".join("    .{ .lo = 0x%X, .hi = 0x%X }" % (lo, hi) for lo, hi in ranges)
82    return (
83        f"pub const {name} = [_]Range{{\n{body},\n}};\n"
84    )
85
86
87def emit_classes(name, ranges):
88    body = ",\n".join(
89        "    .{ .lo = 0x%X, .hi = 0x%X, .class = .%s }" % (lo, hi, prop.lower())
90        for lo, hi, prop in ranges
91    )
92    return f"pub const {name} = [_]ClassRange{{\n{body},\n}};\n"
93
94
95def main():
96    eaw = list(parse(fetch("eaw")))
97    gc = list(parse(fetch("gc")))
98    dcp = list(parse(fetch("dcp")))
99    emoji = list(parse(fetch("emoji")))
100    gbp = list(parse(fetch("gbp")))
101
102    wide = merge(
103        [(lo, hi) for lo, hi, p in eaw if p in ("W", "F")]
104        + [(lo, hi) for lo, hi, p in emoji if p in ("Emoji_Presentation", "Emoji_Modifier_Base")]
105    )
106    zero = merge(
107        [(lo, hi) for lo, hi, p in gc if p in ("Mn", "Me", "Cf")]
108        + [(lo, hi) for lo, hi, p in dcp if p == "Default_Ignorable_Code_Point"]
109        + [(0xFE00, 0xFE0F), (0xE0100, 0xE01EF), (0x200B, 0x200D)]
110    )
111    emoji_prop = select(emoji, ("Emoji", "Emoji_Modifier_Base"))
112    extpict = select(emoji, ("Extended_Pictographic",))
113    incb = list(parse(fetch("dcp"), field=-1))
114    consonant = merge([(lo, hi) for lo, hi, p in incb if p == "Consonant"])
115    linker = merge([(lo, hi) for lo, hi, p in incb if p == "Linker"])
116    gbclasses = classes(
117        gbp,
118        {
119            "CR", "LF", "Control", "Extend", "ZWJ", "Regional_Indicator",
120            "Prepend", "SpacingMark", "L", "V", "T", "LV", "LVT",
121        },
122    )
123
124    version = ""
125    for line in fetch("eaw").splitlines():
126        if line.startswith("# EastAsianWidth-"):
127            version = line.split("-")[1].split(".")[:-1]
128            version = ".".join(version)
129            break
130
131    parts = [
132        "// Generated by tools/gen_width.py from the Unicode Character Database.\n",
133        f"// EastAsianWidth version {version}. Do not edit by hand.\n\n",
134        "pub const Range = struct { lo: u21, hi: u21 };\n",
135        "pub const Class = enum { cr, lf, control, extend, zwj, regional_indicator, prepend, spacingmark, l, v, t, lv, lvt };\n",
136        "pub const ClassRange = struct { lo: u21, hi: u21, class: Class };\n\n",
137        emit_range("wide", wide),
138        "\n",
139        emit_range("zero", zero),
140        "\n",
141        emit_range("emoji", emoji_prop),
142        "\n",
143        emit_range("extended_pictographic", extpict),
144        "\n",
145        emit_range("consonant", consonant),
146        "\n",
147        emit_range("linker", linker),
148        "\n",
149        emit_classes("grapheme", gbclasses),
150    ]
151    out = "".join(parts)
152    with open(sys.argv[1] if len(sys.argv) > 1 else "src/tui/width_tables.zig", "w") as f:
153        f.write(out)
154    print(f"gen_width: wide={len(wide)} zero={len(zero)} emoji={len(emoji_prop)} "
155          f"extpict={len(extpict)} gb={len(gbclasses)} consonant={len(consonant)} linker={len(linker)}")
156
157
158if __name__ == "__main__":
159    main()