@@ -1,5 +1,5 @@
11 import * as ts from "typescript";
2-import type { ClewOutput } from "../model.js";
2+import type { ClewOutput, ClewToken } from "../model.js";
33 import type { ClewLanguageSupport } from "../registry.js";
44 import {
55 createToken,
@@ -10,11 +10,150 @@ import {
1010
1111 const classifier = ts.createClassifier();
1212
13+const GLOBAL_BUILTINS = new Set([
14+ "console",
15+ "globalThis",
16+ "Intl",
17+ "JSON",
18+ "Math",
19+ "Promise",
20+ "Reflect",
21+]);
22+
23+const PARAMETER_PROPERTY_MODIFIERS = new Set([
24+ ts.SyntaxKind.PrivateKeyword,
25+ ts.SyntaxKind.ProtectedKeyword,
26+ ts.SyntaxKind.PublicKeyword,
27+ ts.SyntaxKind.ReadonlyKeyword,
28+ ts.SyntaxKind.OverrideKeyword,
29+]);
30+
1331 interface TokenTypeInfo {
32+ path?: readonly string[];
1433 scopes?: readonly string[];
1534 type: string;
1635 }
1736
37+interface SemanticInfo extends TokenTypeInfo {
38+ priority: number;
39+}
40+
41+const BOOLEAN_INFO = {
42+ priority: 30,
43+ scopes: ["keyword", "constant.boolean"],
44+ type: "keyword",
45+} satisfies SemanticInfo;
46+
47+const BUILTIN_INFO = {
48+ priority: 20,
49+ scopes: ["builtin"],
50+ type: "builtin",
51+} satisfies SemanticInfo;
52+
53+const CLASS_INFO = {
54+ priority: 50,
55+ scopes: ["type", "type.class"],
56+ type: "type",
57+} satisfies SemanticInfo;
58+
59+const DECORATOR_INFO = {
60+ priority: 60,
61+ scopes: ["meta", "meta.decorator"],
62+ type: "meta",
63+} satisfies SemanticInfo;
64+
65+const ENUM_INFO = {
66+ priority: 50,
67+ scopes: ["type", "type.enum"],
68+ type: "type",
69+} satisfies SemanticInfo;
70+
71+const ENUM_MEMBER_INFO = {
72+ priority: 30,
73+ scopes: ["property", "constant.enum"],
74+ type: "property",
75+} satisfies SemanticInfo;
76+
77+const FUNCTION_INFO = {
78+ priority: 40,
79+ scopes: ["function"],
80+ type: "function",
81+} satisfies SemanticInfo;
82+
83+const INTERFACE_INFO = {
84+ priority: 50,
85+ scopes: ["type", "type.interface"],
86+ type: "type",
87+} satisfies SemanticInfo;
88+
89+const NULL_INFO = {
90+ priority: 30,
91+ scopes: ["keyword", "constant.null"],
92+ type: "keyword",
93+} satisfies SemanticInfo;
94+
95+const NUMBER_PROPERTY_INFO = {
96+ priority: 30,
97+ scopes: ["number", "property"],
98+ type: "number",
99+} satisfies SemanticInfo;
100+
101+const PARAMETER_INFO = {
102+ priority: 25,
103+ scopes: ["variable", "variable.parameter"],
104+ type: "variable",
105+} satisfies SemanticInfo;
106+
107+const PRIMITIVE_TYPE_INFO = {
108+ priority: 45,
109+ scopes: ["type", "type.primitive"],
110+ type: "type",
111+} satisfies SemanticInfo;
112+
113+const PROPERTY_INFO = {
114+ priority: 30,
115+ scopes: ["property"],
116+ type: "property",
117+} satisfies SemanticInfo;
118+
119+const STRING_PROPERTY_INFO = {
120+ priority: 30,
121+ scopes: ["string", "property"],
122+ type: "string",
123+} satisfies SemanticInfo;
124+
125+const TYPE_ALIAS_INFO = {
126+ priority: 50,
127+ scopes: ["type", "type.alias"],
128+ type: "type",
129+} satisfies SemanticInfo;
130+
131+const TYPE_INFO = {
132+ priority: 45,
133+ scopes: ["type"],
134+ type: "type",
135+} satisfies SemanticInfo;
136+
137+const TYPE_PARAMETER_INFO = {
138+ priority: 50,
139+ scopes: ["type", "type.parameter"],
140+ type: "type",
141+} satisfies SemanticInfo;
142+
143+const TYPE_KEYWORD_KINDS = new Set([
144+ ts.SyntaxKind.AnyKeyword,
145+ ts.SyntaxKind.BigIntKeyword,
146+ ts.SyntaxKind.BooleanKeyword,
147+ ts.SyntaxKind.NeverKeyword,
148+ ts.SyntaxKind.NumberKeyword,
149+ ts.SyntaxKind.ObjectKeyword,
150+ ts.SyntaxKind.StringKeyword,
151+ ts.SyntaxKind.SymbolKeyword,
152+ ts.SyntaxKind.UndefinedKeyword,
153+ ts.SyntaxKind.UnknownKeyword,
154+ ts.SyntaxKind.VoidKeyword,
155+]);
156+
18157 function tokenInfoFromClassification(classification: number): TokenTypeInfo {
19158 switch (classification) {
20159 case ts.ClassificationType.comment:
@@ -48,8 +187,8 @@ function gapTokenType(text: string): TokenTypeInfo {
48187 return /^\s+$/.test(text) ? { type: "whitespace" } : { type: "text" };
49188 }
50189
51-function tokenizeTypescript(content: string): ClewOutput {
52- const tokens = [];
190+function baseTokensFromTypescript(content: string): ClewToken[] {
191+ const tokens: ClewToken[] = [];
53192 const lines = splitLines(content);
54193 let lineState = ts.EndOfLineState.None;
55194
@@ -88,6 +227,7 @@ function tokenizeTypescript(content: string): ClewOutput {
88227 line.start + start,
89228 info.type,
90229 info.scopes,
230+ info.path,
91231 ),
92232 );
93233 }
@@ -101,6 +241,7 @@ function tokenizeTypescript(content: string): ClewOutput {
101241 line.start + start + length,
102242 info.type,
103243 info.scopes,
244+ info.path,
104245 ),
105246 );
106247 position = start + length;
@@ -116,6 +257,7 @@ function tokenizeTypescript(content: string): ClewOutput {
116257 line.start + line.text.length,
117258 info.type,
118259 info.scopes,
260+ info.path,
119261 ),
120262 );
121263 }
@@ -134,7 +276,386 @@ function tokenizeTypescript(content: string): ClewOutput {
134276 lineState = classifications.endOfLineState;
135277 }
136278
137- return { tokens };
279+ return tokens;
280+}
281+
282+function semanticKey(start: number, end: number): string {
283+ return `${start}:${end}`;
284+}
285+
286+function setSemantic(
287+ semantics: Map<string, SemanticInfo>,
288+ start: number,
289+ end: number,
290+ info: SemanticInfo,
291+): void {
292+ if (end <= start) {
293+ return;
294+ }
295+
296+ const key = semanticKey(start, end);
297+ const existing = semantics.get(key);
298+ if (!existing || info.priority >= existing.priority) {
299+ semantics.set(key, info);
300+ }
301+}
302+
303+function setSemanticForNode(
304+ semantics: Map<string, SemanticInfo>,
305+ sourceFile: ts.SourceFile,
306+ node: ts.Node | undefined,
307+ info: SemanticInfo,
308+): void {
309+ if (!node) {
310+ return;
311+ }
312+
313+ setSemantic(semantics, node.getStart(sourceFile), node.getEnd(), info);
314+}
315+
316+function unwrapExpression(expression: ts.Expression): ts.Expression {
317+ let current = expression;
318+
319+ while (true) {
320+ if (ts.isParenthesizedExpression(current)) {
321+ current = current.expression;
322+ continue;
323+ }
324+ if (ts.isNonNullExpression(current)) {
325+ current = current.expression;
326+ continue;
327+ }
328+ if (ts.isAsExpression(current)) {
329+ current = current.expression;
330+ continue;
331+ }
332+ if (ts.isSatisfiesExpression(current)) {
333+ current = current.expression;
334+ continue;
335+ }
336+ if (ts.isPartiallyEmittedExpression(current)) {
337+ current = current.expression;
338+ continue;
339+ }
340+ return current;
341+ }
342+}
343+
344+function hasParameterPropertyModifier(
345+ parameter: ts.ParameterDeclaration,
346+): boolean {
347+ return (
348+ parameter.modifiers?.some((modifier) =>
349+ PARAMETER_PROPERTY_MODIFIERS.has(modifier.kind),
350+ ) ?? false
351+ );
352+}
353+
354+function semanticForPropertyName(
355+ name: ts.PropertyName | ts.PrivateIdentifier,
356+ info: SemanticInfo,
357+): SemanticInfo {
358+ if (ts.isStringLiteral(name)) {
359+ return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : STRING_PROPERTY_INFO;
360+ }
361+
362+ if (ts.isNumericLiteral(name)) {
363+ return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : NUMBER_PROPERTY_INFO;
364+ }
365+
366+ return info;
367+}
368+
369+function markBindingName(
370+ semantics: Map<string, SemanticInfo>,
371+ sourceFile: ts.SourceFile,
372+ name: ts.BindingName,
373+ info: SemanticInfo,
374+): void {
375+ if (ts.isIdentifier(name)) {
376+ setSemanticForNode(semantics, sourceFile, name, info);
377+ return;
378+ }
379+
380+ for (const element of name.elements) {
381+ if (!ts.isBindingElement(element)) {
382+ continue;
383+ }
384+ markBindingName(semantics, sourceFile, element.name, info);
385+ }
386+}
387+
388+function markPropertyName(
389+ semantics: Map<string, SemanticInfo>,
390+ sourceFile: ts.SourceFile,
391+ name: ts.PropertyName | ts.PrivateIdentifier | undefined,
392+ info = PROPERTY_INFO,
393+): void {
394+ if (!name || ts.isComputedPropertyName(name)) {
395+ return;
396+ }
397+
398+ setSemanticForNode(
399+ semantics,
400+ sourceFile,
401+ name,
402+ semanticForPropertyName(name, info),
403+ );
404+}
405+
406+function markTypeName(
407+ semantics: Map<string, SemanticInfo>,
408+ sourceFile: ts.SourceFile,
409+ node: ts.Node | undefined,
410+ info = TYPE_INFO,
411+): void {
412+ if (!node) {
413+ return;
414+ }
415+
416+ if (ts.isIdentifier(node) || ts.isPrivateIdentifier(node)) {
417+ setSemanticForNode(semantics, sourceFile, node, info);
418+ return;
419+ }
420+
421+ if (ts.isQualifiedName(node)) {
422+ markTypeName(semantics, sourceFile, node.left, info);
423+ markTypeName(semantics, sourceFile, node.right, info);
424+ return;
425+ }
426+
427+ if (ts.isPropertyAccessExpression(node)) {
428+ markTypeName(semantics, sourceFile, node.expression, info);
429+ setSemanticForNode(semantics, sourceFile, node.name, info);
430+ return;
431+ }
432+
433+ if (ts.isImportTypeNode(node) && node.qualifier) {
434+ markTypeName(semantics, sourceFile, node.qualifier, info);
435+ return;
436+ }
437+
438+ if (ts.isThisTypeNode(node)) {
439+ setSemanticForNode(semantics, sourceFile, node, info);
440+ }
441+}
442+
443+function markDecoratorTarget(
444+ semantics: Map<string, SemanticInfo>,
445+ sourceFile: ts.SourceFile,
446+ expression: ts.Expression,
447+): void {
448+ const unwrapped = unwrapExpression(expression);
449+
450+ if (ts.isIdentifier(unwrapped)) {
451+ setSemanticForNode(semantics, sourceFile, unwrapped, DECORATOR_INFO);
452+ return;
453+ }
454+
455+ if (ts.isPropertyAccessExpression(unwrapped)) {
456+ setSemanticForNode(semantics, sourceFile, unwrapped.name, DECORATOR_INFO);
457+ return;
458+ }
459+
460+ if (ts.isCallExpression(unwrapped)) {
461+ markDecoratorTarget(semantics, sourceFile, unwrapped.expression);
462+ }
463+}
464+
465+function markCallTarget(
466+ semantics: Map<string, SemanticInfo>,
467+ sourceFile: ts.SourceFile,
468+ expression: ts.Expression,
469+): void {
470+ const unwrapped = unwrapExpression(expression);
471+
472+ if (ts.isIdentifier(unwrapped)) {
473+ setSemanticForNode(
474+ semantics,
475+ sourceFile,
476+ unwrapped,
477+ GLOBAL_BUILTINS.has(unwrapped.text) ? BUILTIN_INFO : FUNCTION_INFO,
478+ );
479+ return;
480+ }
481+
482+ if (ts.isPropertyAccessExpression(unwrapped)) {
483+ if (
484+ ts.isIdentifier(unwrapped.expression) &&
485+ GLOBAL_BUILTINS.has(unwrapped.expression.text)
486+ ) {
487+ setSemanticForNode(
488+ semantics,
489+ sourceFile,
490+ unwrapped.expression,
491+ BUILTIN_INFO,
492+ );
493+ }
494+
495+ setSemanticForNode(semantics, sourceFile, unwrapped.name, FUNCTION_INFO);
496+ }
497+}
498+
499+function markNewTarget(
500+ semantics: Map<string, SemanticInfo>,
501+ sourceFile: ts.SourceFile,
502+ expression: ts.Expression,
503+): void {
504+ const unwrapped = unwrapExpression(expression);
505+
506+ if (ts.isIdentifier(unwrapped)) {
507+ setSemanticForNode(semantics, sourceFile, unwrapped, TYPE_INFO);
508+ return;
509+ }
510+
511+ if (ts.isPropertyAccessExpression(unwrapped)) {
512+ setSemanticForNode(semantics, sourceFile, unwrapped.name, TYPE_INFO);
513+ }
514+}
515+
516+function collectTypescriptSemantics(
517+ content: string,
518+): Map<string, SemanticInfo> {
519+ const semantics = new Map<string, SemanticInfo>();
520+ const sourceFile = ts.createSourceFile(
521+ "clew.ts",
522+ content,
523+ ts.ScriptTarget.Latest,
524+ true,
525+ ts.ScriptKind.TS,
526+ );
527+
528+ function visit(node: ts.Node): void {
529+ if (TYPE_KEYWORD_KINDS.has(node.kind)) {
530+ setSemanticForNode(semantics, sourceFile, node, PRIMITIVE_TYPE_INFO);
531+ }
532+
533+ switch (node.kind) {
534+ case ts.SyntaxKind.FalseKeyword:
535+ case ts.SyntaxKind.TrueKeyword:
536+ setSemanticForNode(semantics, sourceFile, node, BOOLEAN_INFO);
537+ break;
538+ case ts.SyntaxKind.NullKeyword:
539+ setSemanticForNode(semantics, sourceFile, node, NULL_INFO);
540+ break;
541+ }
542+
543+ if (ts.isDecorator(node)) {
544+ markDecoratorTarget(semantics, sourceFile, node.expression);
545+ } else if (ts.isTypeAliasDeclaration(node)) {
546+ setSemanticForNode(semantics, sourceFile, node.name, TYPE_ALIAS_INFO);
547+ } else if (ts.isInterfaceDeclaration(node)) {
548+ setSemanticForNode(semantics, sourceFile, node.name, INTERFACE_INFO);
549+ } else if (ts.isClassDeclaration(node)) {
550+ setSemanticForNode(semantics, sourceFile, node.name, CLASS_INFO);
551+ } else if (ts.isEnumDeclaration(node)) {
552+ setSemanticForNode(semantics, sourceFile, node.name, ENUM_INFO);
553+ } else if (ts.isModuleDeclaration(node)) {
554+ setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
555+ } else if (ts.isTypeParameterDeclaration(node)) {
556+ setSemanticForNode(semantics, sourceFile, node.name, TYPE_PARAMETER_INFO);
557+ } else if (
558+ ts.isFunctionDeclaration(node) ||
559+ ts.isFunctionExpression(node)
560+ ) {
561+ setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO);
562+ } else if (ts.isMethodDeclaration(node) || ts.isMethodSignature(node)) {
563+ markPropertyName(semantics, sourceFile, node.name, FUNCTION_INFO);
564+ } else if (
565+ ts.isGetAccessorDeclaration(node) ||
566+ ts.isSetAccessorDeclaration(node)
567+ ) {
568+ markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
569+ } else if (ts.isPropertyDeclaration(node) || ts.isPropertySignature(node)) {
570+ markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
571+ } else if (ts.isConstructorDeclaration(node)) {
572+ setSemanticForNode(
573+ semantics,
574+ sourceFile,
575+ node.getFirstToken(sourceFile),
576+ FUNCTION_INFO,
577+ );
578+ } else if (ts.isParameter(node)) {
579+ if (hasParameterPropertyModifier(node)) {
580+ markBindingName(semantics, sourceFile, node.name, PROPERTY_INFO);
581+ } else {
582+ markBindingName(semantics, sourceFile, node.name, PARAMETER_INFO);
583+ }
584+ } else if (ts.isVariableDeclaration(node)) {
585+ if (
586+ ts.isIdentifier(node.name) &&
587+ node.initializer &&
588+ (ts.isArrowFunction(node.initializer) ||
589+ ts.isFunctionExpression(node.initializer))
590+ ) {
591+ setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO);
592+ }
593+ } else if (ts.isPropertyAssignment(node)) {
594+ markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
595+ } else if (ts.isEnumMember(node)) {
596+ markPropertyName(semantics, sourceFile, node.name, ENUM_MEMBER_INFO);
597+ } else if (ts.isPropertyAccessExpression(node)) {
598+ markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
599+ } else if (ts.isCallExpression(node)) {
600+ markCallTarget(semantics, sourceFile, node.expression);
601+ } else if (ts.isTaggedTemplateExpression(node)) {
602+ markCallTarget(semantics, sourceFile, node.tag);
603+ } else if (ts.isNewExpression(node)) {
604+ markNewTarget(semantics, sourceFile, node.expression);
605+ } else if (ts.isTypeReferenceNode(node)) {
606+ markTypeName(semantics, sourceFile, node.typeName);
607+ } else if (ts.isExpressionWithTypeArguments(node)) {
608+ markTypeName(semantics, sourceFile, node.expression);
609+ } else if (ts.isImportTypeNode(node) && node.qualifier) {
610+ markTypeName(semantics, sourceFile, node.qualifier);
611+ } else if (ts.isImportClause(node) && node.isTypeOnly && node.name) {
612+ setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
613+ } else if (ts.isImportSpecifier(node) && node.isTypeOnly) {
614+ setSemanticForNode(
615+ semantics,
616+ sourceFile,
617+ node.propertyName ?? node.name,
618+ TYPE_INFO,
619+ );
620+ setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
621+ }
622+
623+ ts.forEachChild(node, visit);
624+ }
625+
626+ visit(sourceFile);
627+ return semantics;
628+}
629+
630+function applySemantics(
631+ tokens: readonly ClewToken[],
632+ semantics: Map<string, SemanticInfo>,
633+): ClewToken[] {
634+ if (semantics.size === 0) {
635+ return [...tokens];
636+ }
637+
638+ return tokens.map((token) => {
639+ const semantic = semantics.get(semanticKey(token.start, token.end));
640+ if (!semantic) {
641+ return token;
642+ }
643+
644+ return createToken(
645+ token.text,
646+ token.start,
647+ token.end,
648+ semantic.type,
649+ semantic.scopes,
650+ semantic.path,
651+ );
652+ });
653+}
654+
655+function tokenizeTypescript(content: string): ClewOutput {
656+ const baseTokens = baseTokensFromTypescript(content);
657+ const semantics = collectTypescriptSemantics(content);
658+ return { tokens: applySemantics(baseTokens, semantics) };
138659 }
139660
140661 export const typescriptLanguageSupport: ClewLanguageSupport = {