70160
Ce
cel-tui / files
cel-tui / packages / clew / src / index.ts
1.2 KB
1import type { 2 ClewOptions, 3 ClewOutput, 4 ClewStream, 5 ClewStreamOptions, 6 ClewSyncOptions, 7} from "./model.js"; 8import { getLanguageSupport } from "./registry.js"; 9import { createStream } from "./stream.js"; 10 11export type { 12 ClewBaseOptions, 13 ClewChunk, 14 ClewCorrection, 15 ClewOptions, 16 ClewOutput, 17 ClewPatch, 18 ClewStability, 19 ClewStream, 20 ClewStreamOptions, 21 ClewSyncOptions, 22 ClewToken, 23 ClewUnsubscribe, 24} from "./model.js"; 25export { clewSupportsLanguage } from "./registry.js"; 26 27/** 28 * Tokenize full content synchronously. 29 */ 30export function clew(content: string, options: ClewSyncOptions): ClewOutput; 31/** 32 * Create a streaming tokenizer. 33 * 34 * Streaming is the default mode, and `stability` defaults to `"eager"`. 35 * 36 * The current implementation supports TypeScript / JavaScript ids, Python, 37 * Bash, JSON, Markdown, and diff/patch. 38 */ 39export function clew(content: string, options: ClewStreamOptions): ClewStream; 40export function clew( 41 content: string, 42 options: ClewOptions, 43): ClewOutput | ClewStream { 44 const support = getLanguageSupport(options.lang); 45 46 if (options.stream === false) { 47 return support.tokenize(content); 48 } 49 50 return createStream(content, options, support); 51}