70160
Ce
cel-tui / files
cel-tui / packages / clew / src / model.ts
3.5 KB
1/** 2 * A normalized syntax token produced by `clew`. 3 * 4 * Offsets are zero-based UTF-16 code-unit offsets into the source content, 5 * matching JavaScript string indexing. 6 * `start` is inclusive and `end` is exclusive. 7 */ 8export interface ClewToken { 9 /** Token text exactly as it appears in the source. */ 10 text: string; 11 /** Inclusive start offset in the source content. */ 12 start: number; 13 /** Exclusive end offset in the source content. */ 14 end: number; 15 /** Best-effort normalized token kind, such as `keyword` or `string`. */ 16 type: string; 17 /** Optional more specific semantic path. */ 18 path?: readonly string[]; 19 /** Ordered canonical scopes from generic to more specific. */ 20 scopes?: readonly string[]; 21} 22 23/** 24 * The final structured token output for a completed tokenization pass. 25 */ 26export interface ClewOutput { 27 tokens: ClewToken[]; 28} 29 30/** 31 * A token patch emitted during streaming. 32 * 33 * It describes the tokenization for the source range `from..to`. 34 * Consumers can use this to append newly stable output or replace a previously 35 * emitted range with corrected tokens. 36 */ 37export interface ClewPatch { 38 /** Inclusive start offset in the source content. */ 39 from: number; 40 /** Exclusive end offset in the previously emitted source content. */ 41 to: number; 42 /** Replacement tokens for the range. */ 43 tokens: ClewToken[]; 44} 45 46/** 47 * Newly emitted tokenization from the stream. 48 */ 49export type ClewChunk = ClewPatch; 50 51/** 52 * A correction for earlier streamed output. 53 */ 54export type ClewCorrection = ClewPatch; 55 56/** 57 * Removes a previously registered listener. 58 */ 59export type ClewUnsubscribe = () => void; 60 61/** 62 * Controls how aggressively the stream emits tokenization before later input 63 * can confirm it. 64 * 65 * - `eager` — emit as early as possible and correct later if needed. 66 * - `stable` — hold back output until it is less likely to change. 67 */ 68export type ClewStability = "eager" | "stable"; 69 70/** 71 * Common tokenizer options shared by sync and stream modes. 72 */ 73export interface ClewBaseOptions { 74 /** Language identifier used to choose a tokenizer backend or grammar. */ 75 lang: string; 76} 77 78/** 79 * Options for sync tokenization. 80 * 81 * Sync mode is opt-in. `clew` defaults to streaming. 82 */ 83export interface ClewSyncOptions extends ClewBaseOptions { 84 stream: false; 85} 86 87/** 88 * Options for streaming tokenization. 89 */ 90export interface ClewStreamOptions extends ClewBaseOptions { 91 /** Streaming is the default mode. */ 92 stream?: true; 93 /** 94 * Streaming-first default: emit tokenization eagerly and correct it later if 95 * later input changes the parse. 96 */ 97 stability?: ClewStability; 98} 99 100/** 101 * Full options accepted by `clew`. 102 */ 103export type ClewOptions = ClewSyncOptions | ClewStreamOptions; 104 105/** 106 * A stream-first tokenizer interface. 107 * 108 * `write()` accepts additional content, including per-character input. 109 * `onChunk()` emits newly available tokenization. 110 * `onCorrection()` emits range-based corrections when later input changes 111 * earlier tokenization. 112 */ 113export interface ClewStream { 114 /** Register a listener for newly emitted tokenization. */ 115 onChunk(listener: (chunk: ClewChunk) => void): ClewUnsubscribe; 116 /** Register a listener for corrections to earlier tokenization. */ 117 onCorrection(listener: (correction: ClewCorrection) => void): ClewUnsubscribe; 118 /** Write more source content into the stream. */ 119 write(chunk: string): void; 120 /** Return the current token snapshot without ending the stream. */ 121 snapshot(): ClewOutput; 122 /** Finalize the stream and return the final structured output. */ 123 end(): ClewOutput; 124}