79113
Ma
mini-coding-agent / files
mini-coding-agent / tools / gen_width.py
5.3 KB
1#!/usr/bin/env python3 2"""Generates src/tui/width_tables.zig from the Unicode Character Database. 3 4Every table is a sorted list of non-overlapping inclusive codepoint ranges, so 5lookup is a binary search and a normal `zig build` needs no network. Run 6`python3 tools/gen_width.py` to refresh against a newer UCD; the version it 7reads is printed and recorded in the file header. 8""" 9 10import sys 11import urllib.request 12 13BASE = "https://www.unicode.org/Public/UCD/latest/ucd/" 14 15FILES = { 16 "eaw": "EastAsianWidth.txt", 17 "gc": "extracted/DerivedGeneralCategory.txt", 18 "dcp": "DerivedCoreProperties.txt", 19 "emoji": "emoji/emoji-data.txt", 20 "gbp": "auxiliary/GraphemeBreakProperty.txt", 21} 22 23 24def fetch(name): 25 url = BASE + FILES[name] 26 try: 27 with urllib.request.urlopen(url) as r: 28 return r.read().decode("utf-8") 29 except Exception as e: # pragma: no cover 30 sys.exit(f"gen_width: cannot fetch {url}: {e}") 31 32 33def parse(text, field=1): 34 """Yields (lo, hi, property) for every non-comment line of a UCD file. 35 36 A line may carry several semicolon fields; `field` picks one, and -1 picks 37 the last. The InCB lines of DerivedCoreProperties are `range; InCB; Value`, 38 so they are read with field=-1. 39 """ 40 for line in text.splitlines(): 41 line = line.split("#", 1)[0].strip() 42 if not line or ";" not in line: 43 continue 44 parts = [p.strip() for p in line.split(";")] 45 rng = parts[0] 46 lo, _, hi = rng.partition("..") 47 yield int(lo, 16), int(hi or lo, 16), parts[field] 48 49 50def merge(rows): 51 """Sorts and coalesces (lo, hi) pairs into disjoint ranges.""" 52 rows = sorted(set(rows)) 53 out = [] 54 for lo, hi in rows: 55 if out and lo <= out[-1][1] + 1: 56 out[-1][1] = max(out[-1][1], hi) 57 else: 58 out.append([lo, hi]) 59 return [(lo, hi) for lo, hi in out] 60 61 62def select(rows, wanted): 63 return merge([(lo, hi) for lo, hi, prop in rows if prop in wanted]) 64 65 66def classes(rows, wanted): 67 """Merges per-property ranges into a distinct-class range list.""" 68 by_prop = {} 69 for lo, hi, prop in rows: 70 if prop in wanted: 71 by_prop.setdefault(prop, []).append((lo, hi)) 72 out = [] 73 for prop, ranges in by_prop.items(): 74 for lo, hi in merge(ranges): 75 out.append((lo, hi, prop)) 76 out.sort() 77 return out 78 79 80def emit_range(name, ranges): 81 body = ",\n".join(" .{ .lo = 0x%X, .hi = 0x%X }" % (lo, hi) for lo, hi in ranges) 82 return ( 83 f"pub const {name} = [_]Range{{\n{body},\n}};\n" 84 ) 85 86 87def emit_classes(name, ranges): 88 body = ",\n".join( 89 " .{ .lo = 0x%X, .hi = 0x%X, .class = .%s }" % (lo, hi, prop.lower()) 90 for lo, hi, prop in ranges 91 ) 92 return f"pub const {name} = [_]ClassRange{{\n{body},\n}};\n" 93 94 95def main(): 96 eaw = list(parse(fetch("eaw"))) 97 gc = list(parse(fetch("gc"))) 98 dcp = list(parse(fetch("dcp"))) 99 emoji = list(parse(fetch("emoji"))) 100 gbp = list(parse(fetch("gbp"))) 101 102 wide = merge( 103 [(lo, hi) for lo, hi, p in eaw if p in ("W", "F")] 104 + [(lo, hi) for lo, hi, p in emoji if p in ("Emoji_Presentation", "Emoji_Modifier_Base")] 105 ) 106 zero = merge( 107 [(lo, hi) for lo, hi, p in gc if p in ("Mn", "Me", "Cf")] 108 + [(lo, hi) for lo, hi, p in dcp if p == "Default_Ignorable_Code_Point"] 109 + [(0xFE00, 0xFE0F), (0xE0100, 0xE01EF), (0x200B, 0x200D)] 110 ) 111 emoji_prop = select(emoji, ("Emoji", "Emoji_Modifier_Base")) 112 extpict = select(emoji, ("Extended_Pictographic",)) 113 incb = list(parse(fetch("dcp"), field=-1)) 114 consonant = merge([(lo, hi) for lo, hi, p in incb if p == "Consonant"]) 115 linker = merge([(lo, hi) for lo, hi, p in incb if p == "Linker"]) 116 gbclasses = classes( 117 gbp, 118 { 119 "CR", "LF", "Control", "Extend", "ZWJ", "Regional_Indicator", 120 "Prepend", "SpacingMark", "L", "V", "T", "LV", "LVT", 121 }, 122 ) 123 124 version = "" 125 for line in fetch("eaw").splitlines(): 126 if line.startswith("# EastAsianWidth-"): 127 version = line.split("-")[1].split(".")[:-1] 128 version = ".".join(version) 129 break 130 131 parts = [ 132 "// Generated by tools/gen_width.py from the Unicode Character Database.\n", 133 f"// EastAsianWidth version {version}. Do not edit by hand.\n\n", 134 "pub const Range = struct { lo: u21, hi: u21 };\n", 135 "pub const Class = enum { cr, lf, control, extend, zwj, regional_indicator, prepend, spacingmark, l, v, t, lv, lvt };\n", 136 "pub const ClassRange = struct { lo: u21, hi: u21, class: Class };\n\n", 137 emit_range("wide", wide), 138 "\n", 139 emit_range("zero", zero), 140 "\n", 141 emit_range("emoji", emoji_prop), 142 "\n", 143 emit_range("extended_pictographic", extpict), 144 "\n", 145 emit_range("consonant", consonant), 146 "\n", 147 emit_range("linker", linker), 148 "\n", 149 emit_classes("grapheme", gbclasses), 150 ] 151 out = "".join(parts) 152 with open(sys.argv[1] if len(sys.argv) > 1 else "src/tui/width_tables.zig", "w") as f: 153 f.write(out) 154 print(f"gen_width: wide={len(wide)} zero={len(zero)} emoji={len(emoji_prop)} " 155 f"extpict={len(extpict)} gb={len(gbclasses)} consonant={len(consonant)} linker={len(linker)}") 156 157 158if __name__ == "__main__": 159 main()