xonecas.io
6533
Yo

yoctomarkdown / commit f53453d

feat: use string-width for accurate Unicode terminal width calculation

- Replace naive visible-length calculation with string-width + strip-ansi
- Add Intl.Segmenter-based terminalLayout() to handle CJK wide chars,
  emoji ZWJ sequences, combining marks, and ambiguous-width characters
- Add unicode width edge case examples to streaming-test
- Add string-width and strip-ansi dependencies

xonecas2026-03-16 f53453de706ad86b7b46cfd53485d478cb4dcd8a parent 6286b42

4 files changed

bun.lock+13−3
@@ -5,6 +5,8 @@
55 "": {
66 "name": "yoctomarkdown",
77 "dependencies": {
8+ "string-width": "^8.2.0",
9+ "strip-ansi": "^7.2.0",
810 "yoctocolors": "^2.1.2",
911 },
1012 "devDependencies": {
@@ -123,7 +125,7 @@
123125
124126 "acorn": ["acorn@7.4.1", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="],
125127
126- "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
128+ "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
127129
128130 "asap": ["asap@2.0.6", "", {}, "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="],
129131
@@ -189,6 +191,8 @@
189191
190192 "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
191193
194+ "get-east-asian-width": ["get-east-asian-width@1.5.0", "", {}, "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA=="],
195+
192196 "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
193197
194198 "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
@@ -333,9 +337,9 @@
333337
334338 "spark-md5": ["spark-md5@3.0.2", "", {}, "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw=="],
335339
336- "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
340+ "string-width": ["string-width@8.2.0", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-6hJPQ8N0V0P3SNmP6h2J99RLuzrWz2gvT7VnK5tKvrNqJoyS9W4/Fb8mo31UiPvy00z7DQXkP2hnKBVav76thw=="],
337341
338- "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
342+ "strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="],
339343
340344 "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="],
341345
@@ -373,6 +377,12 @@
373377
374378 "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
375379
380+ "cli-table3/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
381+
376382 "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
383+
384+ "cli-table3/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
385+
386+ "cli-table3/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
377387 }
378388 }
examples/streaming-test.ts+31−0
@@ -90,6 +90,37 @@ async function main() {
9090 "A: ",
9191 "A **tiny** streaming markdown highlighter for the terminal.",
9292 );
93+
94+ // 5. Unicode width edge cases — would have been broken before string-width
95+ // CJK wide characters (each takes 2 cells)
96+ await streamWithPrefix(
97+ "CJK: ",
98+ "日本語のテスト — **太字** and `コード` inline.",
99+ );
100+
101+ // Emoji sequences (ZWJ family = 1 grapheme, 2 cells; skin-toned emoji)
102+ await streamWithPrefix(
103+ "Emoji: ",
104+ "Family 👨‍👩‍👧‍👦 and thumbs 👍🏽 and flag 🇯🇵 with **bold** after.",
105+ );
106+
107+ // Ambiguous-width characters (★ ① ☆ ♨ — EAW category "A")
108+ await streamWithPrefix(
109+ "Ambiguous: ",
110+ "Stars ★☆★☆★ and circled ①②③④⑤ and `♨ hotspring` done.",
111+ );
112+
113+ // Combining marks (café with combining é = 4 visible chars)
114+ await streamWithPrefix(
115+ "Combining: ",
116+ "A cafe\u0301 with re\u0301sume\u0301 — **nai\u0308ve** text with `combining` marks.",
117+ );
118+
119+ // Mixed: CJK + emoji + ambiguous + wrapping
120+ await streamWithPrefix(
121+ "Mixed: ",
122+ "漢字と★emoji👨‍👩‍👧‍👦とambiguous①②③が混ざった**長い文**で折り返しテスト。This line mixes scripts to stress-test `visibleLength` on wrapped redraws.",
123+ );
93124 }
94125
95126 main();
package.json+2−0
@@ -15,6 +15,8 @@
1515 "typescript": "^5.9.3"
1616 },
1717 "dependencies": {
18+ "string-width": "^8.2.0",
19+ "strip-ansi": "^7.2.0",
1820 "yoctocolors": "^2.1.2"
1921 },
2022 "bin": {
src/index.ts+28−12
@@ -1,3 +1,5 @@
1+import stripAnsi from "strip-ansi";
2+import stringWidth from "string-width";
13 import { type Theme, themes } from "./themes";
24
35 export interface Options {
@@ -43,6 +45,7 @@ export function highlightSync(input: string, options?: Options): string {
4345 export function createHighlighter(options?: Options): Highlighter {
4446 const theme = resolveTheme(options);
4547 const terminalColumns = resolveColumns(options);
48+ const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
4649 let buffer = "";
4750 let inCodeBlock = false;
4851
@@ -144,10 +147,25 @@ export function createHighlighter(options?: Options): Highlighter {
144147
145148 let previousPartialRendered = "";
146149
147- // Strip ANSI escapes to measure visible character width
148- function visibleLength(s: string): number {
149- // biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape regex
150- return s.replace(/\x1b\[[0-9;]*[A-Za-z]/g, "").length;
150+ // Compute actual terminal line count and last-line column,
151+ // simulating how the terminal wraps wide characters.
152+ // A 2-column char at the last column causes a gap + wrap.
153+ function terminalLayout(s: string): { lines: number; lastCol: number } {
154+ const stripped = stripAnsi(s);
155+ if (stripped.length === 0) return { lines: 1, lastCol: 0 };
156+ let lines = 1;
157+ let col = 0;
158+ for (const { segment } of segmenter.segment(stripped)) {
159+ const w = stringWidth(segment);
160+ if (w === 0) continue;
161+ if (col + w > terminalColumns) {
162+ lines++;
163+ col = w;
164+ } else {
165+ col += w;
166+ }
167+ }
168+ return { lines, lastCol: col };
151169 }
152170
153171 // Erase the previous partial render by moving back over exactly the
@@ -158,17 +176,15 @@ export function createHighlighter(options?: Options): Highlighter {
158176 if (previousPartialRendered.length === 0) {
159177 return "";
160178 }
161- const vLen = visibleLength(previousPartialRendered);
162- const wrappedLines = vLen === 0 ? 0 : Math.ceil(vLen / terminalColumns);
179+ const { lines, lastCol } = terminalLayout(previousPartialRendered);
163180 let seq = "";
164181 // Move cursor up for any extra wrapped lines beyond the first
165- if (wrappedLines > 1) {
166- seq += `\x1b[${wrappedLines - 1}A`;
182+ if (lines > 1) {
183+ seq += `\x1b[${lines - 1}A`;
167184 }
168- // Move cursor back to the start of our partial output.
169- const firstLineLen = wrappedLines > 1 ? terminalColumns : vLen;
170- if (firstLineLen > 0) {
171- seq += `\x1b[${firstLineLen}D`;
185+ // Move cursor back to start of our partial output
186+ if (lastCol > 0) {
187+ seq += `\x1b[${lastCol}D`;
172188 }
173189 // Erase from cursor to end of screen
174190 seq += "\x1b[0J";