xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / components / src / syntax-highlight.ts

20.7 KB
1import {
2  type ClewOutput,
3  type ClewStream,
4  type ClewToken,
5  clew,
6  clewSupportsLanguage,
7} from "@cel-tui/clew";
8import { HStack, Text, VStack, visibleWidth } from "@cel-tui/core";
9import type { Color, ContainerNode, Node, StyleProps } from "@cel-tui/types";
10
11const DEFAULT_THEME_NAME = "cel-ansi16";
12const MAX_CACHED_COLORS = 256;
13const MAX_CACHED_DIRECT_RENDERS = 128;
14const MAX_CACHED_THEMES = 64;
15const TAB_STOP = 4;
16const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
17
18const ANSI_SLOT_HEX: ReadonlyArray<readonly [Color, string]> = [
19  ["color00", "#000000"],
20  ["color01", "#cd3131"],
21  ["color02", "#0dbc79"],
22  ["color03", "#e5e510"],
23  ["color04", "#2472c8"],
24  ["color05", "#bc3fbc"],
25  ["color06", "#11a8cd"],
26  ["color07", "#e5e5e5"],
27  ["color08", "#666666"],
28  ["color09", "#f14c4c"],
29  ["color10", "#23d18b"],
30  ["color11", "#f5f543"],
31  ["color12", "#3b8eea"],
32  ["color13", "#d670d6"],
33  ["color14", "#29b8db"],
34  ["color15", "#ffffff"],
35];
36
37/** A best-effort token color override entry. */
38export interface SyntaxHighlightThemeTokenColor {
39  /** Target canonical `clew` scopes emitted by this component. */
40  readonly scope?: string | readonly string[];
41  /** Style settings for the matched scopes. */
42  readonly settings: {
43    readonly foreground?: string;
44    readonly background?: string;
45    readonly fontStyle?: string;
46  };
47}
48
49/**
50 * Native cel-tui syntax theme fields.
51 *
52 * Palette slots are preserved as `color00`–`color15` references. Changing the
53 * runtime color theme therefore changes the rendered colors without rebuilding
54 * or re-quantizing syntax-highlight nodes.
55 */
56export interface SyntaxHighlightNativeTheme {
57  /** Style inherited by every highlighted token. */
58  readonly baseStyle?: Readonly<StyleProps>;
59  /** Style overrides keyed by canonical `clew` scope. */
60  readonly scopeStyles?: Readonly<Record<string, Readonly<StyleProps>>>;
61}
62
63/**
64 * Theme registration shape accepted by {@link SyntaxHighlight}.
65 * Registrations are immutable values and are cached by object identity and
66 * serialized value. Compatible TextMate fields are applied first; native
67 * {@link SyntaxHighlightNativeTheme} fields take precedence when both target
68 * the same property.
69 */
70export interface SyntaxHighlightThemeRegistration extends SyntaxHighlightNativeTheme {
71  /** Optional theme name. */
72  readonly name?: string;
73  /** Optional light/dark hint. */
74  readonly type?: "light" | "dark";
75  /** Optional base foreground color. */
76  readonly fg?: string;
77  /** Optional base background color. */
78  readonly bg?: string;
79  /** Token color overrides. */
80  readonly tokenColors?: readonly SyntaxHighlightThemeTokenColor[];
81}
82
83/**
84 * SyntaxHighlight theme.
85 *
86 * String presets are intentionally small and stable. Unknown preset names fall
87 * back to the default terminal-friendly theme.
88 */
89export type SyntaxHighlightTheme = string | SyntaxHighlightThemeRegistration;
90
91/** Props for the {@link SyntaxHighlight} component. */
92export interface SyntaxHighlightProps {
93  /**
94   * Optional theme override.
95   *
96   * Built-in presets currently include `"default"` and `"dark-plus"`.
97   * Theme registration objects accept compatible TextMate token colors and/or
98   * native cel-tui `baseStyle` and `scopeStyles` fields. Native fields are
99   * applied last and preserve palette-slot references directly.
100   */
101  theme?: SyntaxHighlightTheme;
102}
103
104/**
105 * A stateful syntax highlighter created by {@link createSyntaxHighlight}.
106 *
107 * Create one instance per independently updating snippet. Calls with appended
108 * content reuse that instance's `clew` stream without sharing parser state
109 * with any other snippet.
110 */
111export interface SyntaxHighlightInstance {
112  /** Render the instance's current source. */
113  (
114    content: string,
115    language: string,
116    props?: SyntaxHighlightProps,
117  ): ContainerNode;
118  /**
119   * Release the cached parser and rendered node.
120   * A later call starts a fresh stream, so the instance remains reusable.
121   */
122  dispose(): void;
123}
124
125interface HighlightRun {
126  content: string;
127  style: StyleProps;
128}
129
130interface ResolvedScopeStyle {
131  style: StyleProps;
132}
133
134interface ResolvedSyntaxHighlightTheme {
135  baseStyle: StyleProps;
136  cacheKey: string;
137  scopeStyles: ReadonlyMap<string, ResolvedScopeStyle>;
138}
139
140interface ClewSyntaxHighlightState {
141  language: string;
142  lastContent: string;
143  lastNode: ContainerNode | null;
144  output: ClewOutput;
145  stream: ClewStream;
146  theme: ResolvedSyntaxHighlightTheme;
147}
148
149const rgbCache = new Map<string, readonly [number, number, number]>();
150const colorSlotCache = new Map<string, Color>();
151const customThemeIdentityCache = new WeakMap<
152  SyntaxHighlightThemeRegistration,
153  ResolvedSyntaxHighlightTheme
154>();
155const customThemeCache = new Map<string, ResolvedSyntaxHighlightTheme>();
156const directRenderCache = new Map<string, ContainerNode>();
157
158type ScopeStyleRecord = Readonly<Record<string, ResolvedScopeStyle>>;
159
160const DEFAULT_SCOPE_STYLES: ScopeStyleRecord = {
161  builtin: { style: { fgColor: "color06" } },
162  command: { style: { fgColor: "color04" } },
163  comment: { style: { fgColor: "color08", italic: true } },
164  escape: { style: { fgColor: "color03" } },
165  function: { style: { fgColor: "color04" } },
166  keyword: { style: { fgColor: "color05" } },
167  link: { style: { fgColor: "color04", underline: true } },
168  meta: { style: { fgColor: "color06" } },
169  "markup.code": { style: { fgColor: "color02" } },
170  "markup.heading": { style: { fgColor: "color06", bold: true } },
171  "markup.list": { style: { fgColor: "color03", bold: true } },
172  "markup.quote": { style: { fgColor: "color02", italic: true } },
173  number: { style: { fgColor: "color03" } },
174  operator: { style: { fgColor: "color05" } },
175  property: { style: { fgColor: "color06" } },
176  regexp: { style: { fgColor: "color01" } },
177  string: { style: { fgColor: "color02" } },
178  type: { style: { fgColor: "color06" } },
179  variable: { style: {} },
180};
181
182const DARK_PLUS_SCOPE_STYLE_OVERRIDES: Readonly<Record<string, StyleProps>> = {
183  builtin: { fgColor: "color06" },
184  command: { fgColor: "color11" },
185  comment: { fgColor: "color08", italic: true },
186  function: { fgColor: "color11" },
187  keyword: { fgColor: "color12" },
188  link: { fgColor: "color14", underline: true },
189  meta: { fgColor: "color08" },
190  "markup.code": { fgColor: "color09" },
191  "markup.heading": { fgColor: "color12", bold: true },
192  "markup.list": { fgColor: "color10", bold: true },
193  "markup.quote": { fgColor: "color08", italic: true },
194  number: { fgColor: "color10" },
195  property: { fgColor: "color07" },
196  regexp: { fgColor: "color09" },
197  string: { fgColor: "color09" },
198  type: { fgColor: "color06" },
199  variable: { fgColor: "color07" },
200};
201
202const DEFAULT_RESOLVED_THEME = buildResolvedTheme(DEFAULT_THEME_NAME, {});
203const DARK_PLUS_RESOLVED_THEME = buildResolvedTheme(
204  "preset:dark-plus",
205  { fgColor: "color07" },
206  DARK_PLUS_SCOPE_STYLE_OVERRIDES,
207);
208
209function requiredAt<T>(
210  items: readonly T[],
211  index: number,
212  description: string,
213): T {
214  const item = items[index];
215  if (item === undefined) {
216    throw new Error(`Missing ${description} at index ${index}`);
217  }
218  return item;
219}
220
221function readLru<K, V>(map: Map<K, V>, key: K): V | undefined {
222  const value = map.get(key);
223  if (value === undefined) {
224    return undefined;
225  }
226
227  map.delete(key);
228  map.set(key, value);
229  return value;
230}
231
232function writeLru<K, V>(
233  map: Map<K, V>,
234  key: K,
235  value: V,
236  maxEntries: number,
237): void {
238  map.delete(key);
239  map.set(key, value);
240
241  while (map.size > maxEntries) {
242    const oldest = map.keys().next();
243    if (oldest.done) {
244      return;
245    }
246    map.delete(oldest.value);
247  }
248}
249
250function normalizeHex(color: string): string {
251  const lower = color.toLowerCase();
252  if (lower.length === 4) {
253    return `#${lower[1]}${lower[1]}${lower[2]}${lower[2]}${lower[3]}${lower[3]}`;
254  }
255  if (lower.length === 9) {
256    return lower.slice(0, 7);
257  }
258  return lower;
259}
260
261function parseHex(color: string): readonly [number, number, number] {
262  const cached = readLru(rgbCache, color);
263  if (cached) {
264    return cached;
265  }
266
267  const normalized = normalizeHex(color);
268  const rgb: readonly [number, number, number] = [
269    Number.parseInt(normalized.slice(1, 3), 16),
270    Number.parseInt(normalized.slice(3, 5), 16),
271    Number.parseInt(normalized.slice(5, 7), 16),
272  ];
273  writeLru(rgbCache, color, rgb, MAX_CACHED_COLORS);
274  return rgb;
275}
276
277function colorDistance(
278  a: readonly [number, number, number],
279  b: readonly [number, number, number],
280): number {
281  const dr = a[0] - b[0];
282  const dg = a[1] - b[1];
283  const db = a[2] - b[2];
284  return dr * dr + dg * dg + db * db;
285}
286
287function colorToSlot(color: string | undefined): Color | undefined {
288  if (!color) {
289    return undefined;
290  }
291
292  const normalized = normalizeHex(color);
293  const cached = readLru(colorSlotCache, normalized);
294  if (cached) {
295    return cached;
296  }
297
298  const rgb = parseHex(normalized);
299  let best = requiredAt(ANSI_SLOT_HEX, 0, "ANSI color slot")[0];
300  let bestDistance = Number.POSITIVE_INFINITY;
301
302  for (const [slot, hex] of ANSI_SLOT_HEX) {
303    const distance = colorDistance(rgb, parseHex(hex));
304    if (distance < bestDistance) {
305      best = slot;
306      bestDistance = distance;
307    }
308  }
309
310  writeLru(colorSlotCache, normalized, best, MAX_CACHED_COLORS);
311  return best;
312}
313
314function mergeStyles(
315  base: Readonly<StyleProps>,
316  overrides: Readonly<StyleProps>,
317): StyleProps {
318  return {
319    fgColor: overrides.fgColor ?? base.fgColor,
320    bgColor: overrides.bgColor ?? base.bgColor,
321    bold: overrides.bold ?? base.bold,
322    italic: overrides.italic ?? base.italic,
323    underline: overrides.underline ?? base.underline,
324  };
325}
326
327function sameStyle(a: StyleProps, b: StyleProps): boolean {
328  return (
329    a.fgColor === b.fgColor &&
330    a.bgColor === b.bgColor &&
331    a.bold === b.bold &&
332    a.italic === b.italic &&
333    a.underline === b.underline
334  );
335}
336
337function themeSettingsToStyle(settings: {
338  background?: string;
339  fontStyle?: string;
340  foreground?: string;
341}): StyleProps {
342  const fontStyle = settings.fontStyle?.trim().toLowerCase();
343  const fontTokens = fontStyle ? fontStyle.split(/\s+/) : [];
344
345  return {
346    fgColor: colorToSlot(settings.foreground),
347    bgColor: colorToSlot(settings.background),
348    bold: fontTokens.includes("bold") || undefined,
349    italic: fontTokens.includes("italic") || undefined,
350    underline: fontTokens.includes("underline") || undefined,
351  };
352}
353
354function cloneScopeStyles(
355  source: ScopeStyleRecord,
356): Map<string, ResolvedScopeStyle> {
357  const scopeStyles = new Map<string, ResolvedScopeStyle>();
358
359  for (const [scope, resolved] of Object.entries(source)) {
360    scopeStyles.set(scope, {
361      style: { ...resolved.style },
362    });
363  }
364
365  return scopeStyles;
366}
367
368function applyScopeStyleOverride(
369  scopeStyles: Map<string, ResolvedScopeStyle>,
370  scope: string,
371  style: Readonly<StyleProps>,
372): void {
373  const current = scopeStyles.get(scope);
374
375  scopeStyles.set(scope, {
376    style: mergeStyles(current?.style ?? {}, style),
377  });
378}
379
380function applyThemeTokenColors(
381  scopeStyles: Map<string, ResolvedScopeStyle>,
382  tokenColors: readonly SyntaxHighlightThemeTokenColor[] | undefined,
383  baseStyle: StyleProps,
384): void {
385  for (const tokenColor of tokenColors ?? []) {
386    const style = themeSettingsToStyle(tokenColor.settings);
387    const scopes = tokenColor.scope
388      ? Array.isArray(tokenColor.scope)
389        ? tokenColor.scope
390        : [tokenColor.scope]
391      : [];
392
393    if (scopes.length === 0) {
394      Object.assign(baseStyle, mergeStyles(baseStyle, style));
395      continue;
396    }
397
398    for (const scope of scopes) {
399      applyScopeStyleOverride(scopeStyles, scope, style);
400    }
401  }
402}
403
404function buildResolvedTheme(
405  cacheKey: string,
406  baseStyle: StyleProps,
407  scopeStyleOverrides?: Readonly<Record<string, Readonly<StyleProps>>>,
408): ResolvedSyntaxHighlightTheme {
409  const scopeStyles = cloneScopeStyles(DEFAULT_SCOPE_STYLES);
410
411  for (const [scope, style] of Object.entries(scopeStyleOverrides ?? {})) {
412    applyScopeStyleOverride(scopeStyles, scope, style);
413  }
414
415  return {
416    baseStyle,
417    cacheKey,
418    scopeStyles,
419  };
420}
421
422function resolveTheme(
423  theme?: SyntaxHighlightTheme,
424): ResolvedSyntaxHighlightTheme {
425  if (!theme || theme === "default") {
426    return DEFAULT_RESOLVED_THEME;
427  }
428
429  if (typeof theme === "string") {
430    if (theme !== "dark-plus") {
431      return DEFAULT_RESOLVED_THEME;
432    }
433
434    return DARK_PLUS_RESOLVED_THEME;
435  }
436
437  const identityCached = customThemeIdentityCache.get(theme);
438  if (identityCached) {
439    return identityCached;
440  }
441
442  const serialized = JSON.stringify(theme);
443  const cached = readLru(customThemeCache, serialized);
444  if (cached) {
445    customThemeIdentityCache.set(theme, cached);
446    return cached;
447  }
448
449  const baseStyle = themeSettingsToStyle({
450    background: theme.bg,
451    foreground: theme.fg,
452  });
453  const scopeStyles = cloneScopeStyles(DEFAULT_SCOPE_STYLES);
454
455  applyThemeTokenColors(scopeStyles, theme.tokenColors, baseStyle);
456
457  Object.assign(baseStyle, mergeStyles(baseStyle, theme.baseStyle ?? {}));
458  for (const [scope, style] of Object.entries(theme.scopeStyles ?? {})) {
459    applyScopeStyleOverride(scopeStyles, scope, style);
460  }
461
462  const resolved = {
463    baseStyle,
464    cacheKey: `custom:${serialized}`,
465    scopeStyles,
466  };
467  writeLru(customThemeCache, serialized, resolved, MAX_CACHED_THEMES);
468  customThemeIdentityCache.set(theme, resolved);
469  return resolved;
470}
471
472function pushRun(
473  line: HighlightRun[],
474  content: string,
475  style: StyleProps,
476): void {
477  if (content.length === 0) {
478    return;
479  }
480
481  const last = line[line.length - 1];
482  if (last && sameStyle(last.style, style)) {
483    last.content += content;
484    return;
485  }
486
487  line.push({ content, style });
488}
489
490function resolveTokenStyle(
491  token: ClewToken,
492  theme: ResolvedSyntaxHighlightTheme,
493): StyleProps {
494  let style = theme.baseStyle;
495  const scopes = token.scopes ?? [token.type];
496
497  for (const scope of scopes) {
498    const scopeStyle = theme.scopeStyles.get(scope);
499    if (!scopeStyle) {
500      continue;
501    }
502    style = mergeStyles(style, scopeStyle.style);
503  }
504
505  return style;
506}
507
508function pushText(
509  lines: HighlightRun[][],
510  content: string,
511  style: StyleProps,
512): void {
513  const pieces = content.split("\n");
514
515  for (let i = 0; i < pieces.length; i++) {
516    pushRun(
517      requiredAt(lines, lines.length - 1, "highlight line"),
518      requiredAt(pieces, i, "text piece"),
519      style,
520    );
521    if (i < pieces.length - 1) {
522      lines.push([]);
523    }
524  }
525}
526
527function tabWidthAtColumn(column: number): number {
528  return TAB_STOP - (column % TAB_STOP);
529}
530
531// Tabs need to be expanded before splitting a line into separate Text nodes,
532// otherwise each HStack child would measure tab width from column zero.
533function expandTabs(
534  text: string,
535  startColumn: number,
536): { text: string; endColumn: number } {
537  if (!text.includes("\t")) {
538    return { text, endColumn: startColumn + visibleWidth(text) };
539  }
540
541  let expanded = "";
542  let column = startColumn;
543
544  for (const { segment } of segmenter.segment(text)) {
545    if (segment === "\t") {
546      const spaces = tabWidthAtColumn(column);
547      expanded += " ".repeat(spaces);
548      column += spaces;
549      continue;
550    }
551
552    expanded += segment;
553    column += visibleWidth(segment);
554  }
555
556  return { text: expanded, endColumn: column };
557}
558
559function runToNodes(
560  run: HighlightRun,
561  startColumn: number,
562): { nodes: Node[]; endColumn: number } {
563  const pieces = run.content.match(/\S+|\s+/g);
564  if (!pieces) {
565    return { nodes: [], endColumn: startColumn };
566  }
567
568  const nodes: Node[] = [];
569  let column = startColumn;
570
571  for (const piece of pieces) {
572    const expanded = expandTabs(piece, column);
573    if (expanded.text.length > 0) {
574      nodes.push(Text(expanded.text, run.style));
575    }
576    column = expanded.endColumn;
577  }
578
579  return { nodes, endColumn: column };
580}
581
582function lineToNode(line: HighlightRun[]): Node {
583  const children: Node[] = [];
584  let column = 0;
585
586  for (const run of line) {
587    const result = runToNodes(run, column);
588    children.push(...result.nodes);
589    column = result.endColumn;
590  }
591
592  return HStack(
593    { flexWrap: "wrap" },
594    children.length > 0 ? children : [Text("")],
595  );
596}
597
598function highlightOutputToLines(
599  output: ClewOutput,
600  theme: ResolvedSyntaxHighlightTheme,
601): HighlightRun[][] {
602  const lines: HighlightRun[][] = [[]];
603
604  for (const token of output.tokens) {
605    pushText(lines, token.text, resolveTokenStyle(token, theme));
606  }
607
608  return lines;
609}
610
611function createState(
612  theme: ResolvedSyntaxHighlightTheme,
613  language: string,
614): ClewSyntaxHighlightState {
615  const stream = clew("", { lang: language, stability: "eager" });
616  return {
617    language,
618    lastContent: "",
619    lastNode: null,
620    output: stream.snapshot(),
621    stream,
622    theme,
623  };
624}
625
626function updateState(state: ClewSyntaxHighlightState, content: string): void {
627  if (content === state.lastContent) {
628    return;
629  }
630
631  if (content.startsWith(state.lastContent)) {
632    const appended = content.slice(state.lastContent.length);
633    if (appended.length > 0) {
634      state.stream.write(appended);
635    }
636  } else {
637    state.stream = clew("", { lang: state.language, stability: "eager" });
638    if (content.length > 0) {
639      state.stream.write(content);
640    }
641  }
642
643  state.output = state.stream.snapshot();
644  state.lastContent = content;
645  state.lastNode = null;
646}
647
648function renderPlainContent(content: string): ContainerNode {
649  const lines = content.split("\n");
650  return VStack(
651    {},
652    lines.map((line) =>
653      lineToNode(line.length > 0 ? [{ content: line, style: {} }] : []),
654    ),
655  );
656}
657
658function renderHighlightedState(
659  state: ClewSyntaxHighlightState,
660): ContainerNode {
661  if (state.lastNode) {
662    return state.lastNode;
663  }
664
665  const lines = highlightOutputToLines(state.output, state.theme);
666  state.lastNode = VStack({}, lines.map(lineToNode));
667  return state.lastNode;
668}
669
670function directRenderKey(
671  content: string,
672  language: string,
673  theme: ResolvedSyntaxHighlightTheme,
674): string {
675  // Length prefixes make the concatenation unambiguous even when source text
676  // contains NULs or other separator-like characters.
677  return `${language.length}:${language}${theme.cacheKey.length}:${theme.cacheKey}${content}`;
678}
679
680function renderDirect(
681  content: string,
682  language: string,
683  theme: ResolvedSyntaxHighlightTheme,
684): ContainerNode {
685  const key = directRenderKey(content, language, theme);
686  const cached = readLru(directRenderCache, key);
687  if (cached) {
688    return cached;
689  }
690
691  const state = createState(theme, language);
692  updateState(state, content);
693  const node = renderHighlightedState(state);
694  writeLru(directRenderCache, key, node, MAX_CACHED_DIRECT_RENDERS);
695  return node;
696}
697
698/**
699 * Create an independently stateful syntax highlighter.
700 *
701 * Use one instance per snippet whose source grows over time. The instance owns
702 * exactly one parser stream, so snippets cannot evict or accidentally reuse
703 * one another's state. Dropping the callable releases it to garbage collection;
704 * call {@link SyntaxHighlightInstance.dispose | dispose} to release it eagerly.
705 *
706 * @returns A callable highlighter with an explicit disposal method.
707 *
708 * @example
709 * ```ts
710 * const highlightLog = createSyntaxHighlight();
711 *
712 * cel.viewport(() =>
713 *   VStack({}, [highlightLog(streamedSource, "typescript")]),
714 * );
715 *
716 * // When the snippet is permanently removed:
717 * highlightLog.dispose();
718 * ```
719 */
720export function createSyntaxHighlight(): SyntaxHighlightInstance {
721  let state: ClewSyntaxHighlightState | undefined;
722
723  function render(
724    content: string,
725    language: string,
726    props?: SyntaxHighlightProps,
727  ): ContainerNode {
728    if (!clewSupportsLanguage(language)) {
729      state = undefined;
730      return renderPlainContent(content);
731    }
732
733    const theme = resolveTheme(props?.theme);
734    if (
735      !state ||
736      state.language !== language ||
737      state.theme.cacheKey !== theme.cacheKey
738    ) {
739      state = createState(theme, language);
740    }
741
742    updateState(state, content);
743    return renderHighlightedState(state);
744  }
745
746  render.dispose = () => {
747    state = undefined;
748  };
749
750  return render as SyntaxHighlightInstance;
751}
752
753/**
754 * Render syntax-highlighted code as cel-tui primitives.
755 *
756 * The component is synchronous and caches exact direct-call renders. Use
757 * {@link createSyntaxHighlight} when a snippet grows over time: the returned
758 * callable owns an isolated `clew` stream and reuses it for appended content.
759 *
760 * Unknown languages render as plain text. The default theme is terminal-friendly
761 * and maps canonical `clew` scopes onto cel's ANSI palette slots while leaving
762 * base text on terminal defaults.
763 *
764 * @param content - Source code to render.
765 * @param language - Registered `clew` language id.
766 * @param props - Optional theme override.
767 * @returns A `VStack` containing one highlighted line per child.
768 */
769export function SyntaxHighlight(
770  content: string,
771  language: string,
772  props?: SyntaxHighlightProps,
773): ContainerNode {
774  if (!clewSupportsLanguage(language)) {
775    return renderPlainContent(content);
776  }
777
778  const theme = resolveTheme(props?.theme);
779  return renderDirect(content, language, theme);
780}