xonecas.io
64875
Mc

mini-coder / commit e80f67a

feat: tree-sitter based markdown rendering.

xonecas2026-09-23 e80f67aa905b46d5fb08e9d80e6ab0df694a8d11 parent 8818339

9 files changed

.npmrc+1−0
@@ -0,0 +1 @@
1+legacy-peer-deps=true
AGENTS.md+1−1
@@ -85,7 +85,7 @@ Hard constraints. Do not cross these without explicit direction:
8585 their use.
8686 - Validate untrusted input once at the boundary with Typebox, then keep internal
8787 code plain-typed.
88-- Ask before adding a dependency. `diff` and `typebox` are approved.
88+- Ask before adding a dependency.
8989 - Do not add tests unless explicitly asked; verify manually.
9090 - Before finishing, review the diff against the direction and guardrails, remove
9191 anything that added scope, and report any requirement you could not satisfy.
package-lock.json+112−0
@@ -10,7 +10,11 @@
1010 "license": "MIT",
1111 "dependencies": {
1212 "@earendil-works/pi-ai": "0.87.0",
13+ "@tree-sitter-grammars/tree-sitter-markdown": "0.3.2",
1314 "diff": "^8.0.2",
15+ "tree-sitter": "0.25.1",
16+ "tree-sitter-javascript": "0.25.0",
17+ "tree-sitter-typescript": "0.23.2",
1418 "typebox": "1.3.27"
1519 },
1620 "bin": {
@@ -593,6 +597,25 @@
593597 "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==",
594598 "license": "MIT"
595599 },
600+ "node_modules/@tree-sitter-grammars/tree-sitter-markdown": {
601+ "version": "0.3.2",
602+ "resolved": "https://registry.npmjs.org/@tree-sitter-grammars/tree-sitter-markdown/-/tree-sitter-markdown-0.3.2.tgz",
603+ "integrity": "sha512-hQXCcDVvg2t4E8cn7zz6jjIBerzk9E9ZlHxJp5IrUOpY4s1YVpXJbMeWZks2/V7lmkPRnnkM8IrTbQ5ltwEOnA==",
604+ "hasInstallScript": true,
605+ "license": "MIT",
606+ "dependencies": {
607+ "node-addon-api": "^8.1.0",
608+ "node-gyp-build": "^4.8.1"
609+ },
610+ "peerDependencies": {
611+ "tree-sitter": "^0.21.1"
612+ },
613+ "peerDependenciesMeta": {
614+ "tree_sitter": {
615+ "optional": true
616+ }
617+ }
618+ },
596619 "node_modules/@types/node": {
597620 "version": "22.20.4",
598621 "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz",
@@ -908,6 +931,15 @@
908931 "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
909932 "license": "MIT"
910933 },
934+ "node_modules/node-addon-api": {
935+ "version": "8.9.2",
936+ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz",
937+ "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==",
938+ "license": "MIT",
939+ "engines": {
940+ "node": "^18 || ^20 || >= 21"
941+ }
942+ },
911943 "node_modules/node-domexception": {
912944 "version": "1.0.0",
913945 "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
@@ -946,6 +978,17 @@
946978 "url": "https://opencollective.com/node-fetch"
947979 }
948980 },
981+ "node_modules/node-gyp-build": {
982+ "version": "4.8.4",
983+ "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
984+ "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
985+ "license": "MIT",
986+ "bin": {
987+ "node-gyp-build": "bin.js",
988+ "node-gyp-build-optional": "optional.js",
989+ "node-gyp-build-test": "build-test.js"
990+ }
991+ },
949992 "node_modules/openai": {
950993 "version": "6.40.0",
951994 "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz",
@@ -1062,6 +1105,75 @@
10621105 "fast-sha256": "^1.3.0"
10631106 }
10641107 },
1108+ "node_modules/tree-sitter": {
1109+ "version": "0.25.1",
1110+ "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.25.1.tgz",
1111+ "integrity": "sha512-mrcEdkYtHfrK1A6fs3O6FxkBo0Qig5XUXqHhxUOQu0bmPo00QF4XaSx4edpazdHwxnSCjlGKGgIqWdaN4dvTLA==",
1112+ "hasInstallScript": true,
1113+ "license": "MIT",
1114+ "dependencies": {
1115+ "node-addon-api": "^8.5.0",
1116+ "node-gyp-build": "^4.8.4"
1117+ }
1118+ },
1119+ "node_modules/tree-sitter-javascript": {
1120+ "version": "0.25.0",
1121+ "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.25.0.tgz",
1122+ "integrity": "sha512-1fCbmzAskZkxcZzN41sFZ2br2iqTYP3tKls1b/HKGNPQUVOpsUxpmGxdN/wMqAk3jYZnYBR1dd/y/0avMeU7dw==",
1123+ "hasInstallScript": true,
1124+ "license": "MIT",
1125+ "dependencies": {
1126+ "node-addon-api": "^8.3.1",
1127+ "node-gyp-build": "^4.8.4"
1128+ },
1129+ "peerDependencies": {
1130+ "tree-sitter": "^0.25.0"
1131+ },
1132+ "peerDependenciesMeta": {
1133+ "tree-sitter": {
1134+ "optional": true
1135+ }
1136+ }
1137+ },
1138+ "node_modules/tree-sitter-typescript": {
1139+ "version": "0.23.2",
1140+ "resolved": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.23.2.tgz",
1141+ "integrity": "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==",
1142+ "hasInstallScript": true,
1143+ "license": "MIT",
1144+ "dependencies": {
1145+ "node-addon-api": "^8.2.2",
1146+ "node-gyp-build": "^4.8.2",
1147+ "tree-sitter-javascript": "^0.23.1"
1148+ },
1149+ "peerDependencies": {
1150+ "tree-sitter": "^0.21.0"
1151+ },
1152+ "peerDependenciesMeta": {
1153+ "tree-sitter": {
1154+ "optional": true
1155+ }
1156+ }
1157+ },
1158+ "node_modules/tree-sitter-typescript/node_modules/tree-sitter-javascript": {
1159+ "version": "0.23.1",
1160+ "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz",
1161+ "integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==",
1162+ "hasInstallScript": true,
1163+ "license": "MIT",
1164+ "dependencies": {
1165+ "node-addon-api": "^8.2.2",
1166+ "node-gyp-build": "^4.8.2"
1167+ },
1168+ "peerDependencies": {
1169+ "tree-sitter": "^0.21.1"
1170+ },
1171+ "peerDependenciesMeta": {
1172+ "tree-sitter": {
1173+ "optional": true
1174+ }
1175+ }
1176+ },
10651177 "node_modules/ts-algebra": {
10661178 "version": "2.0.0",
10671179 "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
package.json+4−0
@@ -13,7 +13,11 @@
1313 },
1414 "dependencies": {
1515 "@earendil-works/pi-ai": "0.87.0",
16+ "@tree-sitter-grammars/tree-sitter-markdown": "0.3.2",
1617 "diff": "^8.0.2",
18+ "tree-sitter": "0.25.1",
19+ "tree-sitter-javascript": "0.25.0",
20+ "tree-sitter-typescript": "0.23.2",
1721 "typebox": "1.3.27"
1822 },
1923 "devDependencies": {
src/highlight.ts+240−0
@@ -0,0 +1,240 @@
1+import { readFileSync } from "node:fs";
2+import { createRequire } from "node:module";
3+import { dirname, join } from "node:path";
4+import Parser from "tree-sitter";
5+import JavaScript from "tree-sitter-javascript";
6+import TypeScript from "tree-sitter-typescript";
7+import Markdown from "@tree-sitter-grammars/tree-sitter-markdown";
8+
9+/**
10+ * Syntax highlighting for committed scrollback, once, over the whole block.
11+ * Grammar queries ship with the grammar packages and are consumed as data, so
12+ * a capture name is the only thing that reaches the palette. Colors are the
13+ * TokyoNight `moon` palette, mapped the way `folke/tokyonight.nvim` maps
14+ * capture names to highlight groups. Truecolor only.
15+ */
16+
17+interface Style {
18+ fg?: string;
19+ bg?: string;
20+ bold?: boolean;
21+ italic?: boolean;
22+ underline?: boolean;
23+}
24+
25+/**
26+ * Headings take their level's color from TokyoNight's rainbow, over a 10%
27+ * tint of it, as `@markup.heading.N.markdown` does. The grammar's own query
28+ * names one heading color for all levels, so the level comes from the marker.
29+ */
30+const HEADINGS: Style[] = [
31+ { fg: "#82aaff", bg: "#2c314a" },
32+ { fg: "#ffc777", bg: "#38343c" },
33+ { fg: "#c3e88d", bg: "#32383f" },
34+ { fg: "#4fd6be", bg: "#263644" },
35+ { fg: "#c099ff", bg: "#32304a" },
36+ { fg: "#fca7ea", bg: "#383148" },
37+].map((style) => ({ ...style, bold: true }));
38+
39+/** Inline code: Helix's `@markup.raw.markdown_inline`, not Neovim's plain one. */
40+const INLINE_CODE: Style = { fg: "#82aaff", bg: "#444a73" };
41+
42+/** Capture name to color. A dotted name falls back to its parent. */
43+const STYLES: Record<string, Style | undefined> = {
44+ comment: { fg: "#636da6" },
45+ constant: { fg: "#ff966c" },
46+ "constant.builtin": { fg: "#65bcff" },
47+ constructor: { fg: "#c099ff" },
48+ escape: { fg: "#c099ff" },
49+ function: { fg: "#82aaff" },
50+ "function.builtin": { fg: "#65bcff" },
51+ "function.method": { fg: "#82aaff" },
52+ keyword: { fg: "#fca7ea" },
53+ number: { fg: "#ff966c" },
54+ operator: { fg: "#89ddff" },
55+ property: { fg: "#4fd6be" },
56+ "punctuation.bracket": { fg: "#828bb8" },
57+ "punctuation.delimiter": { fg: "#89ddff" },
58+ "punctuation.special": { fg: "#89ddff" },
59+ string: { fg: "#c3e88d" },
60+ "string.special": { fg: "#65bcff" },
61+ type: { fg: "#65bcff" },
62+ "type.builtin": { fg: "#589ed7" },
63+ variable: { fg: "#c8d3f5" },
64+ "variable.builtin": { fg: "#ff757f" },
65+ "variable.parameter": { fg: "#ffc777" },
66+ // Markdown's queries predate the `@markup` rename; these are the old names.
67+ "text.emphasis": { italic: true },
68+ "text.strong": { bold: true },
69+ "text.literal": { fg: "#c3e88d" },
70+ "text.uri": { underline: true },
71+ "text.reference": { fg: "#65bcff" },
72+};
73+
74+function styleFor(capture: string): Style | undefined {
75+ for (let name = capture; ; name = name.slice(0, name.lastIndexOf("."))) {
76+ const style = STYLES[name];
77+ if (style !== undefined) return style;
78+ if (!name.includes(".")) return undefined;
79+ }
80+}
81+
82+const require = createRequire(import.meta.url);
83+
84+/** A grammar package keeps its queries next to its sources. */
85+function querySource(pkg: string, ...path: string[]): string {
86+ return readFileSync(join(dirname(require.resolve(`${pkg}/package.json`)), ...path), "utf8");
87+}
88+
89+interface Syntax {
90+ parser: Parser;
91+ query: Parser.Query;
92+}
93+
94+function syntax(language: Parser.Language, ...sources: string[]): Syntax {
95+ const parser = new Parser();
96+ parser.setLanguage(language);
97+ return { parser, query: new Parser.Query(language, sources.join("\n")) };
98+}
99+
100+const JS_QUERIES = querySource("tree-sitter-javascript", "queries", "highlights.scm");
101+const TS_QUERIES = querySource("tree-sitter-typescript", "queries", "highlights.scm");
102+
103+const JAVASCRIPT = syntax(JavaScript, JS_QUERIES);
104+/** TypeScript's own query only covers what it adds to JavaScript. */
105+const TYPESCRIPT = syntax(TypeScript.typescript, JS_QUERIES, TS_QUERIES);
106+const TSX = syntax(TypeScript.tsx, JS_QUERIES, TS_QUERIES);
107+
108+/** Fence info string to grammar, for the languages we carry. */
109+const CODE: Record<string, Syntax | undefined> = {
110+ js: JAVASCRIPT,
111+ javascript: JAVASCRIPT,
112+ jsx: JAVASCRIPT,
113+ ts: TYPESCRIPT,
114+ typescript: TYPESCRIPT,
115+ tsx: TSX,
116+};
117+
118+const MARKDOWN = syntax(
119+ Markdown,
120+ querySource("@tree-sitter-grammars/tree-sitter-markdown", "tree-sitter-markdown", "queries", "highlights.scm"),
121+);
122+const MARKDOWN_INLINE = syntax(
123+ Markdown.inline,
124+ querySource(
125+ "@tree-sitter-grammars/tree-sitter-markdown",
126+ "tree-sitter-markdown-inline",
127+ "queries",
128+ "highlights.scm",
129+ ),
130+);
131+
132+/** One styled range of a line, in byte offsets into that line. */
133+interface Span {
134+ start: number;
135+ end: number;
136+ style: Style;
137+}
138+
139+function spansOf(syntax: Syntax, node: Parser.SyntaxNode, offset = 0): Span[] {
140+ const spans: Span[] = [];
141+ for (const capture of syntax.query.captures(node)) {
142+ const style = styleFor(capture.name);
143+ if (style === undefined) continue;
144+ spans.push({ start: offset + capture.node.startIndex, end: offset + capture.node.endIndex, style });
145+ }
146+ return spans;
147+}
148+
149+function parse(syntax: Syntax, text: string): Parser.SyntaxNode {
150+ return syntax.parser.parse(text).rootNode;
151+}
152+
153+function sgr(style: Style): string {
154+ const channel = (code: number, hex: string): string =>
155+ `${code};2;${[1, 3, 5].map((i) => Number.parseInt(hex.slice(i, i + 2), 16)).join(";")};`;
156+ let out = "\x1b[";
157+ if (style.bold) out += "1;";
158+ if (style.italic) out += "3;";
159+ if (style.underline) out += "4;";
160+ if (style.fg !== undefined) out += channel(38, style.fg);
161+ if (style.bg !== undefined) out += channel(48, style.bg);
162+ return `${out.slice(0, -1)}m`;
163+}
164+
165+/**
166+ * Wraps every span in its color. Spans nest, so the innermost one wins; text
167+ * outside any span is reset to plain. The trailing reset closes the whole text,
168+ * so a span may cross a line break and the last line still ends plain.
169+ */
170+function paint(text: string, spans: Span[]): string {
171+ const events: { at: number; open: boolean; span: Span }[] = [];
172+ for (const span of spans) {
173+ if (span.end <= span.start) continue;
174+ events.push({ at: span.start, open: true, span }, { at: span.end, open: false, span });
175+ }
176+ events.sort((a, b) => a.at - b.at || Number(a.open) - Number(b.open));
177+
178+ const active: Span[] = [];
179+ let out = "";
180+ let plain = true;
181+ let at = 0;
182+ const emit = (end: number): void => {
183+ if (end <= at) return;
184+ const style = active.length === 0 ? undefined : active[active.length - 1].style;
185+ if (style === undefined) {
186+ if (!plain) out += "\x1b[0m";
187+ plain = true;
188+ } else {
189+ out += sgr(style);
190+ plain = false;
191+ }
192+ out += text.slice(at, end);
193+ at = end;
194+ };
195+ for (const event of events) {
196+ emit(event.at);
197+ if (event.open) active.push(event.span);
198+ else active.splice(active.indexOf(event.span), 1);
199+ }
200+ emit(text.length);
201+ return plain ? out : `${out}\x1b[0m`;
202+}
203+
204+/**
205+ * Colors one committed block of Markdown. Inline markup is a separate grammar,
206+ * run over the ranges the block grammar hands over, as its injections do.
207+ */
208+export function highlightMarkdown(text: string): string {
209+ const root = parse(MARKDOWN, text);
210+ let spans = spansOf(MARKDOWN, root);
211+ // Two things the grammar's queries predate: a whole heading is colored by its
212+ // level, and inline code carries a background. Both replace what covers them.
213+ for (const inline of root.descendantsOfType("inline")) {
214+ const inlineRoot = parse(MARKDOWN_INLINE, inline.text);
215+ spans.push(...spansOf(MARKDOWN_INLINE, inlineRoot, inline.startIndex));
216+ for (const code of inlineRoot.descendantsOfType("code_span")) {
217+ spans = recolor(spans, code, INLINE_CODE, inline.startIndex);
218+ }
219+ }
220+ for (const heading of root.descendantsOfType(["atx_heading", "setext_heading"])) {
221+ const marker = heading.namedChildren.find((child) => /^(atx|setext)_h/.test(child.type))?.type;
222+ const level = marker === undefined ? 1 : Number(marker.replace(/\D/g, ""));
223+ spans = recolor(spans, heading, HEADINGS[Math.min(level, HEADINGS.length) - 1]);
224+ }
225+ return paint(text, spans);
226+}
227+
228+/** Replaces every span inside `node` with one span covering the whole node. */
229+function recolor(spans: Span[], node: Parser.SyntaxNode, style: Style, offset = 0): Span[] {
230+ const start = offset + node.startIndex;
231+ const end = offset + node.endIndex;
232+ const kept = spans.filter((span) => span.start < start || span.end > end);
233+ return [...kept, { start, end, style }];
234+}
235+
236+/** Colors a fenced code block's body, by its fence info string. */
237+export function highlightCode(info: string, text: string): string {
238+ const grammar = CODE[info.trim().toLowerCase()];
239+ return grammar === undefined ? text : paint(text, spansOf(grammar, parse(grammar, text)));
240+}
src/tui/stream.ts+75−5
@@ -1,3 +1,4 @@
1+import { highlightCode, highlightMarkdown } from "../highlight.ts";
12 import { dim } from "./styles.ts";
23
34 /** One logical display line: unwrapped text plus the style its rows inherit. */
@@ -22,29 +23,98 @@ export interface StreamRenderer {
2223 reset(): void;
2324 }
2425
25-/** Commits each complete line; the incomplete tail stays in flight. */
26-export class LineStream implements StreamRenderer {
26+/** A fence opening line, and the info string that names its language. */
27+const FENCE_OPEN = /^ {0,3}(`{3,}|~{3,})\s*(\S*)/;
28+const FENCE_CLOSE = /^ {0,3}(`{3,}|~{3,})\s*$/;
29+
30+/**
31+ * Commits each complete line, colored. A fenced code block is held until its
32+ * closing marker arrives, because neither the fence's language nor its body can
33+ * be known before then. A run of prose is held until its block ends, because
34+ * block context is what separates a setext heading from a paragraph and spans
35+ * may cross lines; a blank line, a fence or the end of the turn ends the block.
36+ * The live preview shows the held lines uncolored.
37+ */
38+export class MarkdownStream implements StreamRenderer {
2739 private rest = "";
40+ private fence: { marker: string; info: string; lines: string[] } | null = null;
41+ private prose: string[] = [];
2842
2943 feed(delta: string): BodyLine[] {
3044 this.rest += delta;
3145 const lines = this.rest.split("\n");
3246 this.rest = lines.pop()!;
33- return lines.map((text) => ({ text }));
47+ return lines.flatMap((line) => this.commit(line));
3448 }
3549
3650 pending(): BodyLine[] {
37- return this.rest === "" ? [] : [{ text: this.rest }];
51+ const held = this.fence === null ? this.prose : this.fence.lines;
52+ const rest = this.rest === "" ? [] : [this.rest];
53+ return [...held, ...rest].map((text) => ({ text }));
3854 }
3955
4056 flush(): BodyLine[] {
41- const lines = this.pending();
57+ const fence = this.fence;
58+ const lines = fence === null ? this.releaseProse(this.rest) : this.releaseFence(null);
59+ if (fence !== null && this.rest !== "") lines.push({ text: highlightMarkdown(this.rest) });
4260 this.reset();
4361 return lines;
4462 }
4563
4664 reset(): void {
4765 this.rest = "";
66+ this.fence = null;
67+ this.prose = [];
68+ }
69+
70+ private commit(line: string): BodyLine[] {
71+ const fence = this.fence;
72+ if (fence !== null) {
73+ const close = FENCE_CLOSE.exec(line);
74+ if (close === null || close[1][0] !== fence.marker[0] || close[1].length < fence.marker.length) {
75+ fence.lines.push(line);
76+ return [];
77+ }
78+ return this.releaseFence(line);
79+ }
80+ const open = FENCE_OPEN.exec(line);
81+ if (open !== null) {
82+ this.fence = { marker: open[1], info: open[2], lines: [line] };
83+ return this.releaseProse("");
84+ }
85+ if (line.trim() === "") return [...this.releaseProse(""), { text: line }];
86+ this.prose.push(line);
87+ return [];
88+ }
89+
90+ /**
91+ * Emits the held prose as one text, so the grammar sees whole blocks, then
92+ * hands back the lines. `tail` is the still incomplete line closing the block.
93+ */
94+ private releaseProse(tail: string): BodyLine[] {
95+ const prose = this.prose;
96+ this.prose = [];
97+ if (tail !== "") prose.push(tail);
98+ if (prose.length === 0) return [];
99+ // A block is only a block once its last line ends, so the text needs a
100+ // terminating newline. It is not a line itself: the closing reset `paint`
101+ // leaves after it belongs to the last one.
102+ const lines = highlightMarkdown(`${prose.join("\n")}\n`).split("\n");
103+ const reset = lines.pop()!;
104+ lines[lines.length - 1] += reset;
105+ return lines.map((text) => ({ text }));
106+ }
107+
108+ /** Emits a held block: the fence lines colored, the body by its language. */
109+ private releaseFence(closing: string | null): BodyLine[] {
110+ const fence = this.fence!;
111+ this.fence = null;
112+ const lines: BodyLine[] = [{ text: highlightMarkdown(fence.lines.shift()!) }];
113+ for (const line of highlightCode(fence.info, fence.lines.join("\n")).split("\n")) {
114+ lines.push({ text: line });
115+ }
116+ if (closing !== null) lines.push({ text: highlightMarkdown(closing) });
117+ return lines;
48118 }
49119 }
50120
src/tui/term.ts+12−4
@@ -34,11 +34,22 @@ export function displayWidth(text: string): number {
3434 return width;
3535 }
3636
37+/** An SGR sequence: zero cells wide, so it never affects a row break. */
38+const SGR = /^\x1b\[[0-9;]*m/;
39+
3740 export function expandTabs(text: string, size = 4): string {
3841 if (!text.includes("\t")) return text;
3942 let column = 0;
4043 let out = "";
41- for (const ch of text) {
44+ for (let i = 0; i < text.length; ) {
45+ const escape = SGR.exec(text.slice(i));
46+ if (escape !== null) {
47+ out += escape[0];
48+ i += escape[0].length;
49+ continue;
50+ }
51+ const ch = String.fromCodePoint(text.codePointAt(i)!);
52+ i += ch.length;
4253 if (ch === "\t") {
4354 const spaces = size - (column % size);
4455 out += " ".repeat(spaces);
@@ -51,9 +62,6 @@ export function expandTabs(text: string, size = 4): string {
5162 return out;
5263 }
5364
54-/** An SGR sequence: zero cells wide, so it never affects a row break. */
55-const SGR = /^\x1b\[[0-9;]*m/;
56-
5765 /**
5866 * Splits one logical line into physical rows no wider than `width` cells. SGR
5967 * sequences count as zero cells and stay attached to the text that follows.
src/tui/tui.ts+4−3
@@ -15,7 +15,7 @@ import type { Session } from "../session.ts";
1515 import type { ToolName } from "../config.ts";
1616 import { Terminal, expandTabs, wrapLine, type Key } from "./term.ts";
1717 import { Editor } from "./editor.ts";
18-import { LineStream, TailStream, type BodyLine, type StreamRenderer } from "./stream.ts";
18+import { MarkdownStream, TailStream, type BodyLine, type StreamRenderer } from "./stream.ts";
1919 import { cyan, dim, green, red } from "./styles.ts";
2020 import { contextUsageLine, estimateContextTokens } from "./usage.ts";
2121
@@ -184,7 +184,7 @@ class Tui {
184184 private separator = false;
185185
186186 // In-flight stream state, never persisted: all display-only.
187- private readonly reply: StreamRenderer = new LineStream();
187+ private readonly reply: StreamRenderer = new MarkdownStream();
188188 private readonly activity: StreamRenderer = new TailStream();
189189 private pendingCalls: string[] = [];
190190 private streamed = "";
@@ -438,7 +438,8 @@ class Tui {
438438 .map((block) => block.text)
439439 .join("");
440440 if (text.trim() !== "" && !this.streamed.includes(text)) {
441- this.commitLines(text.trimEnd().split("\n").map((line) => ({ text: line })));
441+ const stream = new MarkdownStream();
442+ this.commitLines([...stream.feed(text.trimEnd()), ...stream.flush()]);
442443 }
443444 this.streamed = "";
444445 }
syntax-highlighting.md+26−0
@@ -0,0 +1,26 @@
1+# We want to support coloring, or highlighting the syntaxes envolved in mini-coder
2+
3+This is a problem that `tree-sitter` was made for, and it's used in `nvim` which I use and can
4+see it's quite an efficient and fast implementation. This should be enough to tokenize the syntaxes.
5+
6+## How
7+
8+The intent is to use tree-sitter in some way to provide the tokenized tree using one of their `Language`s. Then we implement a module that wraps each token in ansi codes for rendering.
9+Some changes to how we render already landed, so we can wait for closing blocks and control when to flush.
10+
11+The live preview stays unstyled. We only highlight once on commit to scrollback. No incremental logic.
12+
13+If this works well, we should also replace the diff coloring we hand rolled and use tree sitter based highlighting as well. No need for two way to highlight.
14+
15+We might add more themes in the future. For now TokyoNight dark only.
16+
17+## Guardrails
18+
19+- Do not format or render the markdown into special chars. Syntax highlighting only
20+
21+## References:
22+
23+- https://github.com/tree-sitter/node-tree-sitter old it seems and likely unmaintained. But seems to be the api we want, string goes in with syntax, tree comes out. No typescript types afaik.
24+- https://github.com/tree-sitter/tree-sitter/tree/master/lib the actual C11 library. We might need it, it has `binding_web` might be what we need, the WASM (last activity 2 weeks ago, seems more alive).
25+- https://github.com/wixdaq is the author of the Popular TokyoNight theme, I want to use this pallete for our default theme.
26+