70160
Ce
cel-tui / files
cel-tui / benchmarks / regression.ts
10.2 KB
1/** 2 * Deterministic regression benchmark harness. 3 * 4 * Each case runs a fixed number of operations for two warmup samples and seven 5 * measured samples. Results are emitted as JSON so commits and machines can be 6 * compared without parsing human-oriented benchmark output. 7 */ 8 9import { cpus } from "node:os"; 10import { CellBuffer, emitBuffer, VStack } from "@cel-tui/core"; 11import { 12 createSyntaxHighlight, 13 SyntaxHighlight, 14} from "../packages/components/src/syntax-highlight.js"; 15import { VirtualList } from "../packages/components/src/virtual-list.js"; 16import { emitDiff } from "../packages/core/src/emitter.js"; 17import { isEditingKey } from "../packages/core/src/keys.js"; 18import { layout } from "../packages/core/src/layout.js"; 19import { paint } from "../packages/core/src/paint.js"; 20import { layoutText } from "../packages/core/src/text-layout.js"; 21import { 22 appTree, 23 inkComparableTree, 24 miniCoderMessage, 25 miniCoderNextConversationTree, 26} from "./helpers.js"; 27 28const WIDTH = 120; 29const HEIGHT = 40; 30const WARMUP_SAMPLES = 2; 31const MEASURED_SAMPLES = 7; 32const LONG_WRAPPED_PARAGRAPH = 33 "The quick brown fox jumps over the lazy dog while terminal cells wrap predictably. ".repeat( 34 110, 35 ); 36const SYNTAX_SOURCES = Array.from({ length: 32 }, (_, snippet) => 37 Array.from( 38 { length: 24 }, 39 (_, line) => `const value${snippet}_${line}: number = ${snippet + line};`, 40 ).join("\n"), 41); 42const FOUR_SYNTAX_SOURCES = SYNTAX_SOURCES.slice(0, 4); 43const FIVE_SYNTAX_SOURCES = SYNTAX_SOURCES.slice(0, 5); 44const BENCHMARK_SYNTAX_THEME = { 45 name: "regression-benchmark", 46 fg: "#e5e5e5", 47 tokenColors: [ 48 { scope: "keyword", settings: { foreground: "#2472c8" } }, 49 { scope: "string", settings: { foreground: "#0dbc79" } }, 50 ], 51} as const; 52const VIRTUAL_MESSAGE_ITEMS = Array.from( 53 { length: 5_000 }, 54 (_, index) => index, 55); 56 57interface BenchmarkDefinition { 58 name: string; 59 iterationsPerSample: number; 60 operation: () => number; 61} 62 63interface BenchmarkResult { 64 name: string; 65 iterationsPerSample: number; 66 medianNsPerOperation: number; 67 rawSamplesNs: number[]; 68} 69 70let benchmarkSink = 0; 71 72function fixedWorkSample(definition: BenchmarkDefinition): number { 73 const start = Bun.nanoseconds(); 74 for (let i = 0; i < definition.iterationsPerSample; i++) { 75 benchmarkSink = (benchmarkSink + definition.operation()) | 0; 76 } 77 return Bun.nanoseconds() - start; 78} 79 80function median(values: readonly number[]): number { 81 const sorted = [...values].sort((a, b) => a - b); 82 const middle = Math.floor(sorted.length / 2); 83 const value = sorted[middle]; 84 if (value === undefined) { 85 throw new Error("Cannot calculate the median of an empty sample set"); 86 } 87 return value; 88} 89 90function measure(definition: BenchmarkDefinition): BenchmarkResult { 91 for (let i = 0; i < WARMUP_SAMPLES; i++) { 92 fixedWorkSample(definition); 93 } 94 95 const rawSamplesNs: number[] = []; 96 for (let i = 0; i < MEASURED_SAMPLES; i++) { 97 rawSamplesNs.push(fixedWorkSample(definition)); 98 } 99 100 return { 101 name: definition.name, 102 iterationsPerSample: definition.iterationsPerSample, 103 medianNsPerOperation: median(rawSamplesNs) / definition.iterationsPerSample, 104 rawSamplesNs, 105 }; 106} 107 108function fullRender( 109 treeFactory: typeof inkComparableTree | (() => ReturnType<typeof appTree>), 110 width: number, 111 height: number, 112): number { 113 const tree = treeFactory(); 114 const layoutTree = layout(tree, width, height); 115 const buffer = new CellBuffer(width, height); 116 paint(layoutTree, buffer); 117 return emitBuffer(buffer).length; 118} 119 120function renderSyntaxFrame(sources: readonly string[]): number { 121 let lineCount = 0; 122 for (const source of sources) { 123 lineCount += SyntaxHighlight(source, "typescript").children.length; 124 } 125 return lineCount; 126} 127 128function renderCustomThemeSyntaxFrame(sources: readonly string[]): number { 129 let lineCount = 0; 130 for (const source of sources) { 131 lineCount += SyntaxHighlight(source, "typescript", { 132 theme: BENCHMARK_SYNTAX_THEME, 133 }).children.length; 134 } 135 return lineCount; 136} 137 138function setConversationScroll( 139 tree: ReturnType<typeof miniCoderNextConversationTree>, 140 scrollOffset: number, 141): void { 142 const conversation = tree.children[0]; 143 if (conversation?.type !== "vstack" && conversation?.type !== "hstack") { 144 throw new Error("Expected the benchmark conversation to be a container"); 145 } 146 conversation.props.scrollOffset = scrollOffset; 147} 148 149function commitSha(): string { 150 const result = Bun.spawnSync(["git", "rev-parse", "HEAD"], { 151 cwd: process.cwd(), 152 stderr: "pipe", 153 stdout: "pipe", 154 }); 155 if (result.exitCode !== 0) return "unknown"; 156 return result.stdout.toString().trim() || "unknown"; 157} 158 159function workingTreeDirty(): boolean { 160 const result = Bun.spawnSync(["git", "status", "--porcelain"], { 161 cwd: process.cwd(), 162 stderr: "pipe", 163 stdout: "pipe", 164 }); 165 return result.exitCode === 0 && result.stdout.length > 0; 166} 167 168const topTree = miniCoderNextConversationTree(1_000); 169setConversationScroll(topTree, 0); 170const topLayout = layout(topTree, WIDTH, HEIGHT); 171 172const bottomTree = miniCoderNextConversationTree(1_000); 173setConversationScroll(bottomTree, Infinity); 174const bottomLayout = layout(bottomTree, WIDTH, HEIGHT); 175 176const identicalLayout = layout(inkComparableTree(), 80, 24); 177const identicalPrevious = new CellBuffer(80, 24); 178const identicalNext = new CellBuffer(80, 24); 179paint(identicalLayout, identicalPrevious); 180paint(identicalLayout, identicalNext); 181 182const syntaxInstances = SYNTAX_SOURCES.map(() => createSyntaxHighlight()); 183for (let index = 0; index < syntaxInstances.length; index++) { 184 syntaxInstances[index]?.(SYNTAX_SOURCES[index] ?? "", "typescript"); 185} 186 187// Prime direct-call entries so these cases measure steady-state frames. The 188// historical four-entry prefix cache was fast at four snippets and thrashed at 189// five; keeping both cases makes that cliff visible in regression output. 190renderSyntaxFrame(FIVE_SYNTAX_SOURCES); 191renderCustomThemeSyntaxFrame(FIVE_SYNTAX_SOURCES); 192 193const virtualConversation = VirtualList<number>({ 194 itemKey: (index) => index, 195 renderItem: miniCoderMessage, 196 estimatedItemHeight: 8, 197 gap: 1, 198 overscan: HEIGHT, 199 defaultStickToBottom: true, 200}); 201 202function renderVirtualConversation(): number { 203 const tree = virtualConversation({ 204 items: VIRTUAL_MESSAGE_ITEMS, 205 width: WIDTH, 206 height: HEIGHT, 207 }); 208 const layoutTree = layout(tree, WIDTH, HEIGHT); 209 return ( 210 virtualConversation.getState().renderedItems + layoutTree.children.length 211 ); 212} 213 214// Prime measured row heights so the regression case represents a steady-state 215// interactive frame rather than first-mount cache population. 216renderVirtualConversation(); 217 218const definitions: BenchmarkDefinition[] = [ 219 { 220 name: "cell_buffer_create_120x40", 221 iterationsPerSample: 400, 222 operation: () => { 223 const buffer = new CellBuffer(WIDTH, HEIGHT); 224 return buffer.width + buffer.height + buffer.get(0, 0).char.length; 225 }, 226 }, 227 { 228 name: "e2e_ink_comparable_80x24", 229 iterationsPerSample: 200, 230 operation: () => fullRender(inkComparableTree, 80, 24), 231 }, 232 { 233 name: "e2e_app_tree_20_120x40", 234 iterationsPerSample: 100, 235 operation: () => fullRender(() => appTree(20), WIDTH, HEIGHT), 236 }, 237 { 238 name: "allocate_and_paint_1000_messages_top_120x40", 239 iterationsPerSample: 8, 240 operation: () => { 241 const buffer = new CellBuffer(WIDTH, HEIGHT); 242 paint(topLayout, buffer); 243 return buffer.get(0, 0).char.length + buffer.get(10, 10).char.length; 244 }, 245 }, 246 { 247 name: "build_and_layout_5000_messages_unvirtualized_120x40", 248 iterationsPerSample: 1, 249 operation: () => { 250 const tree = VStack( 251 { 252 width: WIDTH, 253 height: HEIGHT, 254 gap: 1, 255 overflow: "scroll", 256 scrollOffset: Infinity, 257 }, 258 VIRTUAL_MESSAGE_ITEMS.map(miniCoderMessage), 259 ); 260 return layout(tree, WIDTH, HEIGHT).children.length; 261 }, 262 }, 263 { 264 name: "virtual_list_5000_variable_messages_120x40", 265 iterationsPerSample: 20, 266 operation: renderVirtualConversation, 267 }, 268 { 269 name: "layout_text_ascii_9130_chars_width_80", 270 iterationsPerSample: 10, 271 operation: () => { 272 const result = layoutText(LONG_WRAPPED_PARAGRAPH, 80, "word"); 273 return result.lineCount + (result.lines.at(-1)?.endOffset ?? 0); 274 }, 275 }, 276 { 277 name: "allocate_and_paint_1000_messages_bottom_120x40", 278 iterationsPerSample: 8, 279 operation: () => { 280 const buffer = new CellBuffer(WIDTH, HEIGHT); 281 paint(bottomLayout, buffer); 282 return buffer.get(0, 0).char.length + buffer.get(10, 10).char.length; 283 }, 284 }, 285 { 286 name: "emit_diff_identical_80x24", 287 iterationsPerSample: 400, 288 operation: () => 289 emitDiff(identicalPrevious, identicalNext).length + identicalNext.width, 290 }, 291 { 292 name: "editing_key_lookup_named_and_shortcut", 293 iterationsPerSample: 20_000, 294 operation: () => 295 Number(isEditingKey("enter")) + Number(isEditingKey("ctrl+s")), 296 }, 297 { 298 name: "syntax_highlight_direct_4_static_snippets_frame", 299 iterationsPerSample: 200, 300 operation: () => renderSyntaxFrame(FOUR_SYNTAX_SOURCES), 301 }, 302 { 303 name: "syntax_highlight_direct_5_static_snippets_frame", 304 iterationsPerSample: 200, 305 operation: () => renderSyntaxFrame(FIVE_SYNTAX_SOURCES), 306 }, 307 { 308 name: "syntax_highlight_32_isolated_instances_frame", 309 iterationsPerSample: 100, 310 operation: () => { 311 let lineCount = 0; 312 for (let index = 0; index < syntaxInstances.length; index++) { 313 lineCount += 314 syntaxInstances[index]?.(SYNTAX_SOURCES[index] ?? "", "typescript") 315 .children.length ?? 0; 316 } 317 return lineCount; 318 }, 319 }, 320 { 321 name: "syntax_highlight_cached_custom_theme_5_snippets_frame", 322 iterationsPerSample: 200, 323 operation: () => renderCustomThemeSyntaxFrame(FIVE_SYNTAX_SOURCES), 324 }, 325]; 326 327const output = { 328 schemaVersion: 1, 329 environment: { 330 bunVersion: Bun.version, 331 platform: process.platform, 332 arch: process.arch, 333 cpuModel: cpus()[0]?.model ?? "unknown", 334 commitSha: commitSha(), 335 workingTreeDirty: workingTreeDirty(), 336 }, 337 methodology: { 338 clock: "Bun.nanoseconds", 339 unit: "nanoseconds", 340 statistic: "median", 341 warmupSamples: WARMUP_SAMPLES, 342 measuredSamples: MEASURED_SAMPLES, 343 fixedIterationsPerSample: true, 344 }, 345 cases: definitions.map(measure), 346 checksum: benchmarkSink, 347}; 348 349process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);