xonecas.io
70160
Ce

cel-tui / files

cel-tui / benchmarks / regression.ts

10.2 KB
1/**
2 * Deterministic regression benchmark harness.
3 *
4 * Each case runs a fixed number of operations for two warmup samples and seven
5 * measured samples. Results are emitted as JSON so commits and machines can be
6 * compared without parsing human-oriented benchmark output.
7 */
8
9import { cpus } from "node:os";
10import { CellBuffer, emitBuffer, VStack } from "@cel-tui/core";
11import {
12  createSyntaxHighlight,
13  SyntaxHighlight,
14} from "../packages/components/src/syntax-highlight.js";
15import { VirtualList } from "../packages/components/src/virtual-list.js";
16import { emitDiff } from "../packages/core/src/emitter.js";
17import { isEditingKey } from "../packages/core/src/keys.js";
18import { layout } from "../packages/core/src/layout.js";
19import { paint } from "../packages/core/src/paint.js";
20import { layoutText } from "../packages/core/src/text-layout.js";
21import {
22  appTree,
23  inkComparableTree,
24  miniCoderMessage,
25  miniCoderNextConversationTree,
26} from "./helpers.js";
27
28const WIDTH = 120;
29const HEIGHT = 40;
30const WARMUP_SAMPLES = 2;
31const MEASURED_SAMPLES = 7;
32const LONG_WRAPPED_PARAGRAPH =
33  "The quick brown fox jumps over the lazy dog while terminal cells wrap predictably. ".repeat(
34    110,
35  );
36const SYNTAX_SOURCES = Array.from({ length: 32 }, (_, snippet) =>
37  Array.from(
38    { length: 24 },
39    (_, line) => `const value${snippet}_${line}: number = ${snippet + line};`,
40  ).join("\n"),
41);
42const FOUR_SYNTAX_SOURCES = SYNTAX_SOURCES.slice(0, 4);
43const FIVE_SYNTAX_SOURCES = SYNTAX_SOURCES.slice(0, 5);
44const BENCHMARK_SYNTAX_THEME = {
45  name: "regression-benchmark",
46  fg: "#e5e5e5",
47  tokenColors: [
48    { scope: "keyword", settings: { foreground: "#2472c8" } },
49    { scope: "string", settings: { foreground: "#0dbc79" } },
50  ],
51} as const;
52const VIRTUAL_MESSAGE_ITEMS = Array.from(
53  { length: 5_000 },
54  (_, index) => index,
55);
56
57interface BenchmarkDefinition {
58  name: string;
59  iterationsPerSample: number;
60  operation: () => number;
61}
62
63interface BenchmarkResult {
64  name: string;
65  iterationsPerSample: number;
66  medianNsPerOperation: number;
67  rawSamplesNs: number[];
68}
69
70let benchmarkSink = 0;
71
72function fixedWorkSample(definition: BenchmarkDefinition): number {
73  const start = Bun.nanoseconds();
74  for (let i = 0; i < definition.iterationsPerSample; i++) {
75    benchmarkSink = (benchmarkSink + definition.operation()) | 0;
76  }
77  return Bun.nanoseconds() - start;
78}
79
80function median(values: readonly number[]): number {
81  const sorted = [...values].sort((a, b) => a - b);
82  const middle = Math.floor(sorted.length / 2);
83  const value = sorted[middle];
84  if (value === undefined) {
85    throw new Error("Cannot calculate the median of an empty sample set");
86  }
87  return value;
88}
89
90function measure(definition: BenchmarkDefinition): BenchmarkResult {
91  for (let i = 0; i < WARMUP_SAMPLES; i++) {
92    fixedWorkSample(definition);
93  }
94
95  const rawSamplesNs: number[] = [];
96  for (let i = 0; i < MEASURED_SAMPLES; i++) {
97    rawSamplesNs.push(fixedWorkSample(definition));
98  }
99
100  return {
101    name: definition.name,
102    iterationsPerSample: definition.iterationsPerSample,
103    medianNsPerOperation: median(rawSamplesNs) / definition.iterationsPerSample,
104    rawSamplesNs,
105  };
106}
107
108function fullRender(
109  treeFactory: typeof inkComparableTree | (() => ReturnType<typeof appTree>),
110  width: number,
111  height: number,
112): number {
113  const tree = treeFactory();
114  const layoutTree = layout(tree, width, height);
115  const buffer = new CellBuffer(width, height);
116  paint(layoutTree, buffer);
117  return emitBuffer(buffer).length;
118}
119
120function renderSyntaxFrame(sources: readonly string[]): number {
121  let lineCount = 0;
122  for (const source of sources) {
123    lineCount += SyntaxHighlight(source, "typescript").children.length;
124  }
125  return lineCount;
126}
127
128function renderCustomThemeSyntaxFrame(sources: readonly string[]): number {
129  let lineCount = 0;
130  for (const source of sources) {
131    lineCount += SyntaxHighlight(source, "typescript", {
132      theme: BENCHMARK_SYNTAX_THEME,
133    }).children.length;
134  }
135  return lineCount;
136}
137
138function setConversationScroll(
139  tree: ReturnType<typeof miniCoderNextConversationTree>,
140  scrollOffset: number,
141): void {
142  const conversation = tree.children[0];
143  if (conversation?.type !== "vstack" && conversation?.type !== "hstack") {
144    throw new Error("Expected the benchmark conversation to be a container");
145  }
146  conversation.props.scrollOffset = scrollOffset;
147}
148
149function commitSha(): string {
150  const result = Bun.spawnSync(["git", "rev-parse", "HEAD"], {
151    cwd: process.cwd(),
152    stderr: "pipe",
153    stdout: "pipe",
154  });
155  if (result.exitCode !== 0) return "unknown";
156  return result.stdout.toString().trim() || "unknown";
157}
158
159function workingTreeDirty(): boolean {
160  const result = Bun.spawnSync(["git", "status", "--porcelain"], {
161    cwd: process.cwd(),
162    stderr: "pipe",
163    stdout: "pipe",
164  });
165  return result.exitCode === 0 && result.stdout.length > 0;
166}
167
168const topTree = miniCoderNextConversationTree(1_000);
169setConversationScroll(topTree, 0);
170const topLayout = layout(topTree, WIDTH, HEIGHT);
171
172const bottomTree = miniCoderNextConversationTree(1_000);
173setConversationScroll(bottomTree, Infinity);
174const bottomLayout = layout(bottomTree, WIDTH, HEIGHT);
175
176const identicalLayout = layout(inkComparableTree(), 80, 24);
177const identicalPrevious = new CellBuffer(80, 24);
178const identicalNext = new CellBuffer(80, 24);
179paint(identicalLayout, identicalPrevious);
180paint(identicalLayout, identicalNext);
181
182const syntaxInstances = SYNTAX_SOURCES.map(() => createSyntaxHighlight());
183for (let index = 0; index < syntaxInstances.length; index++) {
184  syntaxInstances[index]?.(SYNTAX_SOURCES[index] ?? "", "typescript");
185}
186
187// Prime direct-call entries so these cases measure steady-state frames. The
188// historical four-entry prefix cache was fast at four snippets and thrashed at
189// five; keeping both cases makes that cliff visible in regression output.
190renderSyntaxFrame(FIVE_SYNTAX_SOURCES);
191renderCustomThemeSyntaxFrame(FIVE_SYNTAX_SOURCES);
192
193const virtualConversation = VirtualList<number>({
194  itemKey: (index) => index,
195  renderItem: miniCoderMessage,
196  estimatedItemHeight: 8,
197  gap: 1,
198  overscan: HEIGHT,
199  defaultStickToBottom: true,
200});
201
202function renderVirtualConversation(): number {
203  const tree = virtualConversation({
204    items: VIRTUAL_MESSAGE_ITEMS,
205    width: WIDTH,
206    height: HEIGHT,
207  });
208  const layoutTree = layout(tree, WIDTH, HEIGHT);
209  return (
210    virtualConversation.getState().renderedItems + layoutTree.children.length
211  );
212}
213
214// Prime measured row heights so the regression case represents a steady-state
215// interactive frame rather than first-mount cache population.
216renderVirtualConversation();
217
218const definitions: BenchmarkDefinition[] = [
219  {
220    name: "cell_buffer_create_120x40",
221    iterationsPerSample: 400,
222    operation: () => {
223      const buffer = new CellBuffer(WIDTH, HEIGHT);
224      return buffer.width + buffer.height + buffer.get(0, 0).char.length;
225    },
226  },
227  {
228    name: "e2e_ink_comparable_80x24",
229    iterationsPerSample: 200,
230    operation: () => fullRender(inkComparableTree, 80, 24),
231  },
232  {
233    name: "e2e_app_tree_20_120x40",
234    iterationsPerSample: 100,
235    operation: () => fullRender(() => appTree(20), WIDTH, HEIGHT),
236  },
237  {
238    name: "allocate_and_paint_1000_messages_top_120x40",
239    iterationsPerSample: 8,
240    operation: () => {
241      const buffer = new CellBuffer(WIDTH, HEIGHT);
242      paint(topLayout, buffer);
243      return buffer.get(0, 0).char.length + buffer.get(10, 10).char.length;
244    },
245  },
246  {
247    name: "build_and_layout_5000_messages_unvirtualized_120x40",
248    iterationsPerSample: 1,
249    operation: () => {
250      const tree = VStack(
251        {
252          width: WIDTH,
253          height: HEIGHT,
254          gap: 1,
255          overflow: "scroll",
256          scrollOffset: Infinity,
257        },
258        VIRTUAL_MESSAGE_ITEMS.map(miniCoderMessage),
259      );
260      return layout(tree, WIDTH, HEIGHT).children.length;
261    },
262  },
263  {
264    name: "virtual_list_5000_variable_messages_120x40",
265    iterationsPerSample: 20,
266    operation: renderVirtualConversation,
267  },
268  {
269    name: "layout_text_ascii_9130_chars_width_80",
270    iterationsPerSample: 10,
271    operation: () => {
272      const result = layoutText(LONG_WRAPPED_PARAGRAPH, 80, "word");
273      return result.lineCount + (result.lines.at(-1)?.endOffset ?? 0);
274    },
275  },
276  {
277    name: "allocate_and_paint_1000_messages_bottom_120x40",
278    iterationsPerSample: 8,
279    operation: () => {
280      const buffer = new CellBuffer(WIDTH, HEIGHT);
281      paint(bottomLayout, buffer);
282      return buffer.get(0, 0).char.length + buffer.get(10, 10).char.length;
283    },
284  },
285  {
286    name: "emit_diff_identical_80x24",
287    iterationsPerSample: 400,
288    operation: () =>
289      emitDiff(identicalPrevious, identicalNext).length + identicalNext.width,
290  },
291  {
292    name: "editing_key_lookup_named_and_shortcut",
293    iterationsPerSample: 20_000,
294    operation: () =>
295      Number(isEditingKey("enter")) + Number(isEditingKey("ctrl+s")),
296  },
297  {
298    name: "syntax_highlight_direct_4_static_snippets_frame",
299    iterationsPerSample: 200,
300    operation: () => renderSyntaxFrame(FOUR_SYNTAX_SOURCES),
301  },
302  {
303    name: "syntax_highlight_direct_5_static_snippets_frame",
304    iterationsPerSample: 200,
305    operation: () => renderSyntaxFrame(FIVE_SYNTAX_SOURCES),
306  },
307  {
308    name: "syntax_highlight_32_isolated_instances_frame",
309    iterationsPerSample: 100,
310    operation: () => {
311      let lineCount = 0;
312      for (let index = 0; index < syntaxInstances.length; index++) {
313        lineCount +=
314          syntaxInstances[index]?.(SYNTAX_SOURCES[index] ?? "", "typescript")
315            .children.length ?? 0;
316      }
317      return lineCount;
318    },
319  },
320  {
321    name: "syntax_highlight_cached_custom_theme_5_snippets_frame",
322    iterationsPerSample: 200,
323    operation: () => renderCustomThemeSyntaxFrame(FIVE_SYNTAX_SOURCES),
324  },
325];
326
327const output = {
328  schemaVersion: 1,
329  environment: {
330    bunVersion: Bun.version,
331    platform: process.platform,
332    arch: process.arch,
333    cpuModel: cpus()[0]?.model ?? "unknown",
334    commitSha: commitSha(),
335    workingTreeDirty: workingTreeDirty(),
336  },
337  methodology: {
338    clock: "Bun.nanoseconds",
339    unit: "nanoseconds",
340    statistic: "median",
341    warmupSamples: WARMUP_SAMPLES,
342    measuredSamples: MEASURED_SAMPLES,
343    fixedIterationsPerSample: true,
344  },
345  cases: definitions.map(measure),
346  checksum: benchmarkSink,
347};
348
349process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);