70160
Ce
cel-tui / files
cel-tui / packages / clew / src / languages / typescript.ts
17.2 KB
1import * as ts from "typescript"; 2import type { ClewOutput, ClewToken } from "../model.js"; 3import type { ClewLanguageSupport } from "../registry.js"; 4import { 5 createToken, 6 requiredAt, 7 splitLines, 8 stableLineBoundary, 9} from "../shared.js"; 10 11const classifier = ts.createClassifier(); 12 13const GLOBAL_BUILTINS = new Set([ 14 "console", 15 "globalThis", 16 "Intl", 17 "JSON", 18 "Math", 19 "Promise", 20 "Reflect", 21]); 22 23const PARAMETER_PROPERTY_MODIFIERS = new Set([ 24 ts.SyntaxKind.PrivateKeyword, 25 ts.SyntaxKind.ProtectedKeyword, 26 ts.SyntaxKind.PublicKeyword, 27 ts.SyntaxKind.ReadonlyKeyword, 28 ts.SyntaxKind.OverrideKeyword, 29]); 30 31interface TokenTypeInfo { 32 path?: readonly string[]; 33 scopes?: readonly string[]; 34 type: string; 35} 36 37interface SemanticInfo extends TokenTypeInfo { 38 priority: number; 39} 40 41const BOOLEAN_INFO = { 42 priority: 30, 43 scopes: ["keyword", "constant.boolean"], 44 type: "keyword", 45} satisfies SemanticInfo; 46 47const BUILTIN_INFO = { 48 priority: 20, 49 scopes: ["builtin"], 50 type: "builtin", 51} satisfies SemanticInfo; 52 53const CLASS_INFO = { 54 priority: 50, 55 scopes: ["type", "type.class"], 56 type: "type", 57} satisfies SemanticInfo; 58 59const DECORATOR_INFO = { 60 priority: 60, 61 scopes: ["meta", "meta.decorator"], 62 type: "meta", 63} satisfies SemanticInfo; 64 65const ENUM_INFO = { 66 priority: 50, 67 scopes: ["type", "type.enum"], 68 type: "type", 69} satisfies SemanticInfo; 70 71const ENUM_MEMBER_INFO = { 72 priority: 30, 73 scopes: ["property", "constant.enum"], 74 type: "property", 75} satisfies SemanticInfo; 76 77const FUNCTION_INFO = { 78 priority: 40, 79 scopes: ["function"], 80 type: "function", 81} satisfies SemanticInfo; 82 83const INTERFACE_INFO = { 84 priority: 50, 85 scopes: ["type", "type.interface"], 86 type: "type", 87} satisfies SemanticInfo; 88 89const NULL_INFO = { 90 priority: 30, 91 scopes: ["keyword", "constant.null"], 92 type: "keyword", 93} satisfies SemanticInfo; 94 95const NUMBER_PROPERTY_INFO = { 96 priority: 30, 97 scopes: ["number", "property"], 98 type: "number", 99} satisfies SemanticInfo; 100 101const PARAMETER_INFO = { 102 priority: 25, 103 scopes: ["variable", "variable.parameter"], 104 type: "variable", 105} satisfies SemanticInfo; 106 107const PRIMITIVE_TYPE_INFO = { 108 priority: 45, 109 scopes: ["type", "type.primitive"], 110 type: "type", 111} satisfies SemanticInfo; 112 113const PROPERTY_INFO = { 114 priority: 30, 115 scopes: ["property"], 116 type: "property", 117} satisfies SemanticInfo; 118 119const STRING_PROPERTY_INFO = { 120 priority: 30, 121 scopes: ["string", "property"], 122 type: "string", 123} satisfies SemanticInfo; 124 125const TYPE_ALIAS_INFO = { 126 priority: 50, 127 scopes: ["type", "type.alias"], 128 type: "type", 129} satisfies SemanticInfo; 130 131const TYPE_INFO = { 132 priority: 45, 133 scopes: ["type"], 134 type: "type", 135} satisfies SemanticInfo; 136 137const TYPE_PARAMETER_INFO = { 138 priority: 50, 139 scopes: ["type", "type.parameter"], 140 type: "type", 141} satisfies SemanticInfo; 142 143const TYPE_KEYWORD_KINDS = new Set([ 144 ts.SyntaxKind.AnyKeyword, 145 ts.SyntaxKind.BigIntKeyword, 146 ts.SyntaxKind.BooleanKeyword, 147 ts.SyntaxKind.NeverKeyword, 148 ts.SyntaxKind.NumberKeyword, 149 ts.SyntaxKind.ObjectKeyword, 150 ts.SyntaxKind.StringKeyword, 151 ts.SyntaxKind.SymbolKeyword, 152 ts.SyntaxKind.UndefinedKeyword, 153 ts.SyntaxKind.UnknownKeyword, 154 ts.SyntaxKind.VoidKeyword, 155]); 156 157function tokenInfoFromClassification(classification: number): TokenTypeInfo { 158 switch (classification) { 159 case ts.ClassificationType.comment: 160 return { type: "comment" }; 161 case ts.ClassificationType.identifier: 162 return { type: "identifier", scopes: ["identifier", "variable"] }; 163 case ts.ClassificationType.keyword: 164 return { type: "keyword" }; 165 case ts.ClassificationType.numericLiteral: 166 return { type: "number" }; 167 case ts.ClassificationType.bigintLiteral: 168 return { type: "number", scopes: ["number", "number.bigint"] }; 169 case ts.ClassificationType.operator: 170 return { type: "operator" }; 171 case ts.ClassificationType.stringLiteral: 172 return { type: "string" }; 173 case ts.ClassificationType.regularExpressionLiteral: 174 return { type: "regexp" }; 175 case ts.ClassificationType.whiteSpace: 176 return { type: "whitespace" }; 177 case ts.ClassificationType.punctuation: 178 return { type: "punctuation" }; 179 case ts.ClassificationType.text: 180 return { type: "text" }; 181 default: 182 return { type: "identifier", scopes: ["identifier", "variable"] }; 183 } 184} 185 186function gapTokenType(text: string): TokenTypeInfo { 187 return /^\s+$/.test(text) ? { type: "whitespace" } : { type: "text" }; 188} 189 190function baseTokensFromTypescript(content: string): ClewToken[] { 191 const tokens: ClewToken[] = []; 192 const lines = splitLines(content); 193 let lineState = ts.EndOfLineState.None; 194 195 for (const line of lines) { 196 const classifications = classifier.getEncodedLexicalClassifications( 197 line.text, 198 lineState, 199 true, 200 ); 201 202 let position = 0; 203 for (let i = 0; i < classifications.spans.length; i += 3) { 204 const start = requiredAt( 205 classifications.spans, 206 i, 207 "classification start", 208 ); 209 const length = requiredAt( 210 classifications.spans, 211 i + 1, 212 "classification length", 213 ); 214 const classification = requiredAt( 215 classifications.spans, 216 i + 2, 217 "classification kind", 218 ); 219 220 if (start > position) { 221 const text = line.text.slice(position, start); 222 const info = gapTokenType(text); 223 tokens.push( 224 createToken( 225 text, 226 line.start + position, 227 line.start + start, 228 info.type, 229 info.scopes, 230 info.path, 231 ), 232 ); 233 } 234 235 const tokenText = line.text.slice(start, start + length); 236 const info = tokenInfoFromClassification(classification); 237 tokens.push( 238 createToken( 239 tokenText, 240 line.start + start, 241 line.start + start + length, 242 info.type, 243 info.scopes, 244 info.path, 245 ), 246 ); 247 position = start + length; 248 } 249 250 if (position < line.text.length) { 251 const text = line.text.slice(position); 252 const info = gapTokenType(text); 253 tokens.push( 254 createToken( 255 text, 256 line.start + position, 257 line.start + line.text.length, 258 info.type, 259 info.scopes, 260 info.path, 261 ), 262 ); 263 } 264 265 if (line.delimiter.length > 0) { 266 tokens.push( 267 createToken( 268 line.delimiter, 269 line.start + line.text.length, 270 line.start + line.text.length + line.delimiter.length, 271 "whitespace", 272 ), 273 ); 274 } 275 276 lineState = classifications.endOfLineState; 277 } 278 279 return tokens; 280} 281 282function semanticKey(start: number, end: number): string { 283 return `${start}:${end}`; 284} 285 286function setSemantic( 287 semantics: Map<string, SemanticInfo>, 288 start: number, 289 end: number, 290 info: SemanticInfo, 291): void { 292 if (end <= start) { 293 return; 294 } 295 296 const key = semanticKey(start, end); 297 const existing = semantics.get(key); 298 if (!existing || info.priority >= existing.priority) { 299 semantics.set(key, info); 300 } 301} 302 303function setSemanticForNode( 304 semantics: Map<string, SemanticInfo>, 305 sourceFile: ts.SourceFile, 306 node: ts.Node | undefined, 307 info: SemanticInfo, 308): void { 309 if (!node) { 310 return; 311 } 312 313 setSemantic(semantics, node.getStart(sourceFile), node.getEnd(), info); 314} 315 316function unwrapExpression(expression: ts.Expression): ts.Expression { 317 let current = expression; 318 319 while (true) { 320 if (ts.isParenthesizedExpression(current)) { 321 current = current.expression; 322 continue; 323 } 324 if (ts.isNonNullExpression(current)) { 325 current = current.expression; 326 continue; 327 } 328 if (ts.isAsExpression(current)) { 329 current = current.expression; 330 continue; 331 } 332 if (ts.isSatisfiesExpression(current)) { 333 current = current.expression; 334 continue; 335 } 336 if (ts.isPartiallyEmittedExpression(current)) { 337 current = current.expression; 338 continue; 339 } 340 return current; 341 } 342} 343 344function hasParameterPropertyModifier( 345 parameter: ts.ParameterDeclaration, 346): boolean { 347 return ( 348 parameter.modifiers?.some((modifier) => 349 PARAMETER_PROPERTY_MODIFIERS.has(modifier.kind), 350 ) ?? false 351 ); 352} 353 354function semanticForPropertyName( 355 name: ts.PropertyName | ts.PrivateIdentifier, 356 info: SemanticInfo, 357): SemanticInfo { 358 if (ts.isStringLiteral(name)) { 359 return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : STRING_PROPERTY_INFO; 360 } 361 362 if (ts.isNumericLiteral(name)) { 363 return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : NUMBER_PROPERTY_INFO; 364 } 365 366 return info; 367} 368 369function markBindingName( 370 semantics: Map<string, SemanticInfo>, 371 sourceFile: ts.SourceFile, 372 name: ts.BindingName, 373 info: SemanticInfo, 374): void { 375 if (ts.isIdentifier(name)) { 376 setSemanticForNode(semantics, sourceFile, name, info); 377 return; 378 } 379 380 for (const element of name.elements) { 381 if (!ts.isBindingElement(element)) { 382 continue; 383 } 384 markBindingName(semantics, sourceFile, element.name, info); 385 } 386} 387 388function markPropertyName( 389 semantics: Map<string, SemanticInfo>, 390 sourceFile: ts.SourceFile, 391 name: ts.PropertyName | ts.PrivateIdentifier | undefined, 392 info = PROPERTY_INFO, 393): void { 394 if (!name || ts.isComputedPropertyName(name)) { 395 return; 396 } 397 398 setSemanticForNode( 399 semantics, 400 sourceFile, 401 name, 402 semanticForPropertyName(name, info), 403 ); 404} 405 406function markTypeName( 407 semantics: Map<string, SemanticInfo>, 408 sourceFile: ts.SourceFile, 409 node: ts.Node | undefined, 410 info = TYPE_INFO, 411): void { 412 if (!node) { 413 return; 414 } 415 416 if (ts.isIdentifier(node) || ts.isPrivateIdentifier(node)) { 417 setSemanticForNode(semantics, sourceFile, node, info); 418 return; 419 } 420 421 if (ts.isQualifiedName(node)) { 422 markTypeName(semantics, sourceFile, node.left, info); 423 markTypeName(semantics, sourceFile, node.right, info); 424 return; 425 } 426 427 if (ts.isPropertyAccessExpression(node)) { 428 markTypeName(semantics, sourceFile, node.expression, info); 429 setSemanticForNode(semantics, sourceFile, node.name, info); 430 return; 431 } 432 433 if (ts.isImportTypeNode(node) && node.qualifier) { 434 markTypeName(semantics, sourceFile, node.qualifier, info); 435 return; 436 } 437 438 if (ts.isThisTypeNode(node)) { 439 setSemanticForNode(semantics, sourceFile, node, info); 440 } 441} 442 443function markDecoratorTarget( 444 semantics: Map<string, SemanticInfo>, 445 sourceFile: ts.SourceFile, 446 expression: ts.Expression, 447): void { 448 const unwrapped = unwrapExpression(expression); 449 450 if (ts.isIdentifier(unwrapped)) { 451 setSemanticForNode(semantics, sourceFile, unwrapped, DECORATOR_INFO); 452 return; 453 } 454 455 if (ts.isPropertyAccessExpression(unwrapped)) { 456 setSemanticForNode(semantics, sourceFile, unwrapped.name, DECORATOR_INFO); 457 return; 458 } 459 460 if (ts.isCallExpression(unwrapped)) { 461 markDecoratorTarget(semantics, sourceFile, unwrapped.expression); 462 } 463} 464 465function markCallTarget( 466 semantics: Map<string, SemanticInfo>, 467 sourceFile: ts.SourceFile, 468 expression: ts.Expression, 469): void { 470 const unwrapped = unwrapExpression(expression); 471 472 if (ts.isIdentifier(unwrapped)) { 473 setSemanticForNode( 474 semantics, 475 sourceFile, 476 unwrapped, 477 GLOBAL_BUILTINS.has(unwrapped.text) ? BUILTIN_INFO : FUNCTION_INFO, 478 ); 479 return; 480 } 481 482 if (ts.isPropertyAccessExpression(unwrapped)) { 483 if ( 484 ts.isIdentifier(unwrapped.expression) && 485 GLOBAL_BUILTINS.has(unwrapped.expression.text) 486 ) { 487 setSemanticForNode( 488 semantics, 489 sourceFile, 490 unwrapped.expression, 491 BUILTIN_INFO, 492 ); 493 } 494 495 setSemanticForNode(semantics, sourceFile, unwrapped.name, FUNCTION_INFO); 496 } 497} 498 499function markNewTarget( 500 semantics: Map<string, SemanticInfo>, 501 sourceFile: ts.SourceFile, 502 expression: ts.Expression, 503): void { 504 const unwrapped = unwrapExpression(expression); 505 506 if (ts.isIdentifier(unwrapped)) { 507 setSemanticForNode(semantics, sourceFile, unwrapped, TYPE_INFO); 508 return; 509 } 510 511 if (ts.isPropertyAccessExpression(unwrapped)) { 512 setSemanticForNode(semantics, sourceFile, unwrapped.name, TYPE_INFO); 513 } 514} 515 516function collectTypescriptSemantics( 517 content: string, 518): Map<string, SemanticInfo> { 519 const semantics = new Map<string, SemanticInfo>(); 520 const sourceFile = ts.createSourceFile( 521 "clew.ts", 522 content, 523 ts.ScriptTarget.Latest, 524 true, 525 ts.ScriptKind.TS, 526 ); 527 528 function visit(node: ts.Node): void { 529 if (TYPE_KEYWORD_KINDS.has(node.kind)) { 530 setSemanticForNode(semantics, sourceFile, node, PRIMITIVE_TYPE_INFO); 531 } 532 533 switch (node.kind) { 534 case ts.SyntaxKind.FalseKeyword: 535 case ts.SyntaxKind.TrueKeyword: 536 setSemanticForNode(semantics, sourceFile, node, BOOLEAN_INFO); 537 break; 538 case ts.SyntaxKind.NullKeyword: 539 setSemanticForNode(semantics, sourceFile, node, NULL_INFO); 540 break; 541 } 542 543 if (ts.isDecorator(node)) { 544 markDecoratorTarget(semantics, sourceFile, node.expression); 545 } else if (ts.isTypeAliasDeclaration(node)) { 546 setSemanticForNode(semantics, sourceFile, node.name, TYPE_ALIAS_INFO); 547 } else if (ts.isInterfaceDeclaration(node)) { 548 setSemanticForNode(semantics, sourceFile, node.name, INTERFACE_INFO); 549 } else if (ts.isClassDeclaration(node)) { 550 setSemanticForNode(semantics, sourceFile, node.name, CLASS_INFO); 551 } else if (ts.isEnumDeclaration(node)) { 552 setSemanticForNode(semantics, sourceFile, node.name, ENUM_INFO); 553 } else if (ts.isModuleDeclaration(node)) { 554 setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO); 555 } else if (ts.isTypeParameterDeclaration(node)) { 556 setSemanticForNode(semantics, sourceFile, node.name, TYPE_PARAMETER_INFO); 557 } else if ( 558 ts.isFunctionDeclaration(node) || 559 ts.isFunctionExpression(node) 560 ) { 561 setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO); 562 } else if (ts.isMethodDeclaration(node) || ts.isMethodSignature(node)) { 563 markPropertyName(semantics, sourceFile, node.name, FUNCTION_INFO); 564 } else if ( 565 ts.isGetAccessorDeclaration(node) || 566 ts.isSetAccessorDeclaration(node) 567 ) { 568 markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO); 569 } else if (ts.isPropertyDeclaration(node) || ts.isPropertySignature(node)) { 570 markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO); 571 } else if (ts.isConstructorDeclaration(node)) { 572 setSemanticForNode( 573 semantics, 574 sourceFile, 575 node.getFirstToken(sourceFile), 576 FUNCTION_INFO, 577 ); 578 } else if (ts.isParameter(node)) { 579 if (hasParameterPropertyModifier(node)) { 580 markBindingName(semantics, sourceFile, node.name, PROPERTY_INFO); 581 } else { 582 markBindingName(semantics, sourceFile, node.name, PARAMETER_INFO); 583 } 584 } else if (ts.isVariableDeclaration(node)) { 585 if ( 586 ts.isIdentifier(node.name) && 587 node.initializer && 588 (ts.isArrowFunction(node.initializer) || 589 ts.isFunctionExpression(node.initializer)) 590 ) { 591 setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO); 592 } 593 } else if (ts.isPropertyAssignment(node)) { 594 markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO); 595 } else if (ts.isEnumMember(node)) { 596 markPropertyName(semantics, sourceFile, node.name, ENUM_MEMBER_INFO); 597 } else if (ts.isPropertyAccessExpression(node)) { 598 markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO); 599 } else if (ts.isCallExpression(node)) { 600 markCallTarget(semantics, sourceFile, node.expression); 601 } else if (ts.isTaggedTemplateExpression(node)) { 602 markCallTarget(semantics, sourceFile, node.tag); 603 } else if (ts.isNewExpression(node)) { 604 markNewTarget(semantics, sourceFile, node.expression); 605 } else if (ts.isTypeReferenceNode(node)) { 606 markTypeName(semantics, sourceFile, node.typeName); 607 } else if (ts.isExpressionWithTypeArguments(node)) { 608 markTypeName(semantics, sourceFile, node.expression); 609 } else if (ts.isImportTypeNode(node) && node.qualifier) { 610 markTypeName(semantics, sourceFile, node.qualifier); 611 } else if (ts.isImportClause(node) && node.isTypeOnly && node.name) { 612 setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO); 613 } else if (ts.isImportSpecifier(node) && node.isTypeOnly) { 614 setSemanticForNode( 615 semantics, 616 sourceFile, 617 node.propertyName ?? node.name, 618 TYPE_INFO, 619 ); 620 setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO); 621 } 622 623 ts.forEachChild(node, visit); 624 } 625 626 visit(sourceFile); 627 return semantics; 628} 629 630function applySemantics( 631 tokens: readonly ClewToken[], 632 semantics: Map<string, SemanticInfo>, 633): ClewToken[] { 634 if (semantics.size === 0) { 635 return [...tokens]; 636 } 637 638 return tokens.map((token) => { 639 const semantic = semantics.get(semanticKey(token.start, token.end)); 640 if (!semantic) { 641 return token; 642 } 643 644 return createToken( 645 token.text, 646 token.start, 647 token.end, 648 semantic.type, 649 semantic.scopes, 650 semantic.path, 651 ); 652 }); 653} 654 655function tokenizeTypescript(content: string): ClewOutput { 656 const baseTokens = baseTokensFromTypescript(content); 657 const semantics = collectTypescriptSemantics(content); 658 return { tokens: applySemantics(baseTokens, semantics) }; 659} 660 661export const typescriptLanguageSupport: ClewLanguageSupport = { 662 ids: ["typescript", "ts", "mts", "cts", "javascript", "js", "mjs", "cjs"], 663 stableBoundary: stableLineBoundary, 664 tokenize: tokenizeTypescript, 665};