xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / model.ts

3.5 KB
1/**
2 * A normalized syntax token produced by `clew`.
3 *
4 * Offsets are zero-based UTF-16 code-unit offsets into the source content,
5 * matching JavaScript string indexing.
6 * `start` is inclusive and `end` is exclusive.
7 */
8export interface ClewToken {
9  /** Token text exactly as it appears in the source. */
10  text: string;
11  /** Inclusive start offset in the source content. */
12  start: number;
13  /** Exclusive end offset in the source content. */
14  end: number;
15  /** Best-effort normalized token kind, such as `keyword` or `string`. */
16  type: string;
17  /** Optional more specific semantic path. */
18  path?: readonly string[];
19  /** Ordered canonical scopes from generic to more specific. */
20  scopes?: readonly string[];
21}
22
23/**
24 * The final structured token output for a completed tokenization pass.
25 */
26export interface ClewOutput {
27  tokens: ClewToken[];
28}
29
30/**
31 * A token patch emitted during streaming.
32 *
33 * It describes the tokenization for the source range `from..to`.
34 * Consumers can use this to append newly stable output or replace a previously
35 * emitted range with corrected tokens.
36 */
37export interface ClewPatch {
38  /** Inclusive start offset in the source content. */
39  from: number;
40  /** Exclusive end offset in the previously emitted source content. */
41  to: number;
42  /** Replacement tokens for the range. */
43  tokens: ClewToken[];
44}
45
46/**
47 * Newly emitted tokenization from the stream.
48 */
49export type ClewChunk = ClewPatch;
50
51/**
52 * A correction for earlier streamed output.
53 */
54export type ClewCorrection = ClewPatch;
55
56/**
57 * Removes a previously registered listener.
58 */
59export type ClewUnsubscribe = () => void;
60
61/**
62 * Controls how aggressively the stream emits tokenization before later input
63 * can confirm it.
64 *
65 * - `eager` — emit as early as possible and correct later if needed.
66 * - `stable` — hold back output until it is less likely to change.
67 */
68export type ClewStability = "eager" | "stable";
69
70/**
71 * Common tokenizer options shared by sync and stream modes.
72 */
73export interface ClewBaseOptions {
74  /** Language identifier used to choose a tokenizer backend or grammar. */
75  lang: string;
76}
77
78/**
79 * Options for sync tokenization.
80 *
81 * Sync mode is opt-in. `clew` defaults to streaming.
82 */
83export interface ClewSyncOptions extends ClewBaseOptions {
84  stream: false;
85}
86
87/**
88 * Options for streaming tokenization.
89 */
90export interface ClewStreamOptions extends ClewBaseOptions {
91  /** Streaming is the default mode. */
92  stream?: true;
93  /**
94   * Streaming-first default: emit tokenization eagerly and correct it later if
95   * later input changes the parse.
96   */
97  stability?: ClewStability;
98}
99
100/**
101 * Full options accepted by `clew`.
102 */
103export type ClewOptions = ClewSyncOptions | ClewStreamOptions;
104
105/**
106 * A stream-first tokenizer interface.
107 *
108 * `write()` accepts additional content, including per-character input.
109 * `onChunk()` emits newly available tokenization.
110 * `onCorrection()` emits range-based corrections when later input changes
111 * earlier tokenization.
112 */
113export interface ClewStream {
114  /** Register a listener for newly emitted tokenization. */
115  onChunk(listener: (chunk: ClewChunk) => void): ClewUnsubscribe;
116  /** Register a listener for corrections to earlier tokenization. */
117  onCorrection(listener: (correction: ClewCorrection) => void): ClewUnsubscribe;
118  /** Write more source content into the stream. */
119  write(chunk: string): void;
120  /** Return the current token snapshot without ending the stream. */
121  snapshot(): ClewOutput;
122  /** Finalize the stream and return the final structured output. */
123  end(): ClewOutput;
124}