xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / languages / typescript.ts

17.2 KB
1import * as ts from "typescript";
2import type { ClewOutput, ClewToken } from "../model.js";
3import type { ClewLanguageSupport } from "../registry.js";
4import {
5  createToken,
6  requiredAt,
7  splitLines,
8  stableLineBoundary,
9} from "../shared.js";
10
11const classifier = ts.createClassifier();
12
13const GLOBAL_BUILTINS = new Set([
14  "console",
15  "globalThis",
16  "Intl",
17  "JSON",
18  "Math",
19  "Promise",
20  "Reflect",
21]);
22
23const PARAMETER_PROPERTY_MODIFIERS = new Set([
24  ts.SyntaxKind.PrivateKeyword,
25  ts.SyntaxKind.ProtectedKeyword,
26  ts.SyntaxKind.PublicKeyword,
27  ts.SyntaxKind.ReadonlyKeyword,
28  ts.SyntaxKind.OverrideKeyword,
29]);
30
31interface TokenTypeInfo {
32  path?: readonly string[];
33  scopes?: readonly string[];
34  type: string;
35}
36
37interface SemanticInfo extends TokenTypeInfo {
38  priority: number;
39}
40
41const BOOLEAN_INFO = {
42  priority: 30,
43  scopes: ["keyword", "constant.boolean"],
44  type: "keyword",
45} satisfies SemanticInfo;
46
47const BUILTIN_INFO = {
48  priority: 20,
49  scopes: ["builtin"],
50  type: "builtin",
51} satisfies SemanticInfo;
52
53const CLASS_INFO = {
54  priority: 50,
55  scopes: ["type", "type.class"],
56  type: "type",
57} satisfies SemanticInfo;
58
59const DECORATOR_INFO = {
60  priority: 60,
61  scopes: ["meta", "meta.decorator"],
62  type: "meta",
63} satisfies SemanticInfo;
64
65const ENUM_INFO = {
66  priority: 50,
67  scopes: ["type", "type.enum"],
68  type: "type",
69} satisfies SemanticInfo;
70
71const ENUM_MEMBER_INFO = {
72  priority: 30,
73  scopes: ["property", "constant.enum"],
74  type: "property",
75} satisfies SemanticInfo;
76
77const FUNCTION_INFO = {
78  priority: 40,
79  scopes: ["function"],
80  type: "function",
81} satisfies SemanticInfo;
82
83const INTERFACE_INFO = {
84  priority: 50,
85  scopes: ["type", "type.interface"],
86  type: "type",
87} satisfies SemanticInfo;
88
89const NULL_INFO = {
90  priority: 30,
91  scopes: ["keyword", "constant.null"],
92  type: "keyword",
93} satisfies SemanticInfo;
94
95const NUMBER_PROPERTY_INFO = {
96  priority: 30,
97  scopes: ["number", "property"],
98  type: "number",
99} satisfies SemanticInfo;
100
101const PARAMETER_INFO = {
102  priority: 25,
103  scopes: ["variable", "variable.parameter"],
104  type: "variable",
105} satisfies SemanticInfo;
106
107const PRIMITIVE_TYPE_INFO = {
108  priority: 45,
109  scopes: ["type", "type.primitive"],
110  type: "type",
111} satisfies SemanticInfo;
112
113const PROPERTY_INFO = {
114  priority: 30,
115  scopes: ["property"],
116  type: "property",
117} satisfies SemanticInfo;
118
119const STRING_PROPERTY_INFO = {
120  priority: 30,
121  scopes: ["string", "property"],
122  type: "string",
123} satisfies SemanticInfo;
124
125const TYPE_ALIAS_INFO = {
126  priority: 50,
127  scopes: ["type", "type.alias"],
128  type: "type",
129} satisfies SemanticInfo;
130
131const TYPE_INFO = {
132  priority: 45,
133  scopes: ["type"],
134  type: "type",
135} satisfies SemanticInfo;
136
137const TYPE_PARAMETER_INFO = {
138  priority: 50,
139  scopes: ["type", "type.parameter"],
140  type: "type",
141} satisfies SemanticInfo;
142
143const TYPE_KEYWORD_KINDS = new Set([
144  ts.SyntaxKind.AnyKeyword,
145  ts.SyntaxKind.BigIntKeyword,
146  ts.SyntaxKind.BooleanKeyword,
147  ts.SyntaxKind.NeverKeyword,
148  ts.SyntaxKind.NumberKeyword,
149  ts.SyntaxKind.ObjectKeyword,
150  ts.SyntaxKind.StringKeyword,
151  ts.SyntaxKind.SymbolKeyword,
152  ts.SyntaxKind.UndefinedKeyword,
153  ts.SyntaxKind.UnknownKeyword,
154  ts.SyntaxKind.VoidKeyword,
155]);
156
157function tokenInfoFromClassification(classification: number): TokenTypeInfo {
158  switch (classification) {
159    case ts.ClassificationType.comment:
160      return { type: "comment" };
161    case ts.ClassificationType.identifier:
162      return { type: "identifier", scopes: ["identifier", "variable"] };
163    case ts.ClassificationType.keyword:
164      return { type: "keyword" };
165    case ts.ClassificationType.numericLiteral:
166      return { type: "number" };
167    case ts.ClassificationType.bigintLiteral:
168      return { type: "number", scopes: ["number", "number.bigint"] };
169    case ts.ClassificationType.operator:
170      return { type: "operator" };
171    case ts.ClassificationType.stringLiteral:
172      return { type: "string" };
173    case ts.ClassificationType.regularExpressionLiteral:
174      return { type: "regexp" };
175    case ts.ClassificationType.whiteSpace:
176      return { type: "whitespace" };
177    case ts.ClassificationType.punctuation:
178      return { type: "punctuation" };
179    case ts.ClassificationType.text:
180      return { type: "text" };
181    default:
182      return { type: "identifier", scopes: ["identifier", "variable"] };
183  }
184}
185
186function gapTokenType(text: string): TokenTypeInfo {
187  return /^\s+$/.test(text) ? { type: "whitespace" } : { type: "text" };
188}
189
190function baseTokensFromTypescript(content: string): ClewToken[] {
191  const tokens: ClewToken[] = [];
192  const lines = splitLines(content);
193  let lineState = ts.EndOfLineState.None;
194
195  for (const line of lines) {
196    const classifications = classifier.getEncodedLexicalClassifications(
197      line.text,
198      lineState,
199      true,
200    );
201
202    let position = 0;
203    for (let i = 0; i < classifications.spans.length; i += 3) {
204      const start = requiredAt(
205        classifications.spans,
206        i,
207        "classification start",
208      );
209      const length = requiredAt(
210        classifications.spans,
211        i + 1,
212        "classification length",
213      );
214      const classification = requiredAt(
215        classifications.spans,
216        i + 2,
217        "classification kind",
218      );
219
220      if (start > position) {
221        const text = line.text.slice(position, start);
222        const info = gapTokenType(text);
223        tokens.push(
224          createToken(
225            text,
226            line.start + position,
227            line.start + start,
228            info.type,
229            info.scopes,
230            info.path,
231          ),
232        );
233      }
234
235      const tokenText = line.text.slice(start, start + length);
236      const info = tokenInfoFromClassification(classification);
237      tokens.push(
238        createToken(
239          tokenText,
240          line.start + start,
241          line.start + start + length,
242          info.type,
243          info.scopes,
244          info.path,
245        ),
246      );
247      position = start + length;
248    }
249
250    if (position < line.text.length) {
251      const text = line.text.slice(position);
252      const info = gapTokenType(text);
253      tokens.push(
254        createToken(
255          text,
256          line.start + position,
257          line.start + line.text.length,
258          info.type,
259          info.scopes,
260          info.path,
261        ),
262      );
263    }
264
265    if (line.delimiter.length > 0) {
266      tokens.push(
267        createToken(
268          line.delimiter,
269          line.start + line.text.length,
270          line.start + line.text.length + line.delimiter.length,
271          "whitespace",
272        ),
273      );
274    }
275
276    lineState = classifications.endOfLineState;
277  }
278
279  return tokens;
280}
281
282function semanticKey(start: number, end: number): string {
283  return `${start}:${end}`;
284}
285
286function setSemantic(
287  semantics: Map<string, SemanticInfo>,
288  start: number,
289  end: number,
290  info: SemanticInfo,
291): void {
292  if (end <= start) {
293    return;
294  }
295
296  const key = semanticKey(start, end);
297  const existing = semantics.get(key);
298  if (!existing || info.priority >= existing.priority) {
299    semantics.set(key, info);
300  }
301}
302
303function setSemanticForNode(
304  semantics: Map<string, SemanticInfo>,
305  sourceFile: ts.SourceFile,
306  node: ts.Node | undefined,
307  info: SemanticInfo,
308): void {
309  if (!node) {
310    return;
311  }
312
313  setSemantic(semantics, node.getStart(sourceFile), node.getEnd(), info);
314}
315
316function unwrapExpression(expression: ts.Expression): ts.Expression {
317  let current = expression;
318
319  while (true) {
320    if (ts.isParenthesizedExpression(current)) {
321      current = current.expression;
322      continue;
323    }
324    if (ts.isNonNullExpression(current)) {
325      current = current.expression;
326      continue;
327    }
328    if (ts.isAsExpression(current)) {
329      current = current.expression;
330      continue;
331    }
332    if (ts.isSatisfiesExpression(current)) {
333      current = current.expression;
334      continue;
335    }
336    if (ts.isPartiallyEmittedExpression(current)) {
337      current = current.expression;
338      continue;
339    }
340    return current;
341  }
342}
343
344function hasParameterPropertyModifier(
345  parameter: ts.ParameterDeclaration,
346): boolean {
347  return (
348    parameter.modifiers?.some((modifier) =>
349      PARAMETER_PROPERTY_MODIFIERS.has(modifier.kind),
350    ) ?? false
351  );
352}
353
354function semanticForPropertyName(
355  name: ts.PropertyName | ts.PrivateIdentifier,
356  info: SemanticInfo,
357): SemanticInfo {
358  if (ts.isStringLiteral(name)) {
359    return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : STRING_PROPERTY_INFO;
360  }
361
362  if (ts.isNumericLiteral(name)) {
363    return info === ENUM_MEMBER_INFO ? ENUM_MEMBER_INFO : NUMBER_PROPERTY_INFO;
364  }
365
366  return info;
367}
368
369function markBindingName(
370  semantics: Map<string, SemanticInfo>,
371  sourceFile: ts.SourceFile,
372  name: ts.BindingName,
373  info: SemanticInfo,
374): void {
375  if (ts.isIdentifier(name)) {
376    setSemanticForNode(semantics, sourceFile, name, info);
377    return;
378  }
379
380  for (const element of name.elements) {
381    if (!ts.isBindingElement(element)) {
382      continue;
383    }
384    markBindingName(semantics, sourceFile, element.name, info);
385  }
386}
387
388function markPropertyName(
389  semantics: Map<string, SemanticInfo>,
390  sourceFile: ts.SourceFile,
391  name: ts.PropertyName | ts.PrivateIdentifier | undefined,
392  info = PROPERTY_INFO,
393): void {
394  if (!name || ts.isComputedPropertyName(name)) {
395    return;
396  }
397
398  setSemanticForNode(
399    semantics,
400    sourceFile,
401    name,
402    semanticForPropertyName(name, info),
403  );
404}
405
406function markTypeName(
407  semantics: Map<string, SemanticInfo>,
408  sourceFile: ts.SourceFile,
409  node: ts.Node | undefined,
410  info = TYPE_INFO,
411): void {
412  if (!node) {
413    return;
414  }
415
416  if (ts.isIdentifier(node) || ts.isPrivateIdentifier(node)) {
417    setSemanticForNode(semantics, sourceFile, node, info);
418    return;
419  }
420
421  if (ts.isQualifiedName(node)) {
422    markTypeName(semantics, sourceFile, node.left, info);
423    markTypeName(semantics, sourceFile, node.right, info);
424    return;
425  }
426
427  if (ts.isPropertyAccessExpression(node)) {
428    markTypeName(semantics, sourceFile, node.expression, info);
429    setSemanticForNode(semantics, sourceFile, node.name, info);
430    return;
431  }
432
433  if (ts.isImportTypeNode(node) && node.qualifier) {
434    markTypeName(semantics, sourceFile, node.qualifier, info);
435    return;
436  }
437
438  if (ts.isThisTypeNode(node)) {
439    setSemanticForNode(semantics, sourceFile, node, info);
440  }
441}
442
443function markDecoratorTarget(
444  semantics: Map<string, SemanticInfo>,
445  sourceFile: ts.SourceFile,
446  expression: ts.Expression,
447): void {
448  const unwrapped = unwrapExpression(expression);
449
450  if (ts.isIdentifier(unwrapped)) {
451    setSemanticForNode(semantics, sourceFile, unwrapped, DECORATOR_INFO);
452    return;
453  }
454
455  if (ts.isPropertyAccessExpression(unwrapped)) {
456    setSemanticForNode(semantics, sourceFile, unwrapped.name, DECORATOR_INFO);
457    return;
458  }
459
460  if (ts.isCallExpression(unwrapped)) {
461    markDecoratorTarget(semantics, sourceFile, unwrapped.expression);
462  }
463}
464
465function markCallTarget(
466  semantics: Map<string, SemanticInfo>,
467  sourceFile: ts.SourceFile,
468  expression: ts.Expression,
469): void {
470  const unwrapped = unwrapExpression(expression);
471
472  if (ts.isIdentifier(unwrapped)) {
473    setSemanticForNode(
474      semantics,
475      sourceFile,
476      unwrapped,
477      GLOBAL_BUILTINS.has(unwrapped.text) ? BUILTIN_INFO : FUNCTION_INFO,
478    );
479    return;
480  }
481
482  if (ts.isPropertyAccessExpression(unwrapped)) {
483    if (
484      ts.isIdentifier(unwrapped.expression) &&
485      GLOBAL_BUILTINS.has(unwrapped.expression.text)
486    ) {
487      setSemanticForNode(
488        semantics,
489        sourceFile,
490        unwrapped.expression,
491        BUILTIN_INFO,
492      );
493    }
494
495    setSemanticForNode(semantics, sourceFile, unwrapped.name, FUNCTION_INFO);
496  }
497}
498
499function markNewTarget(
500  semantics: Map<string, SemanticInfo>,
501  sourceFile: ts.SourceFile,
502  expression: ts.Expression,
503): void {
504  const unwrapped = unwrapExpression(expression);
505
506  if (ts.isIdentifier(unwrapped)) {
507    setSemanticForNode(semantics, sourceFile, unwrapped, TYPE_INFO);
508    return;
509  }
510
511  if (ts.isPropertyAccessExpression(unwrapped)) {
512    setSemanticForNode(semantics, sourceFile, unwrapped.name, TYPE_INFO);
513  }
514}
515
516function collectTypescriptSemantics(
517  content: string,
518): Map<string, SemanticInfo> {
519  const semantics = new Map<string, SemanticInfo>();
520  const sourceFile = ts.createSourceFile(
521    "clew.ts",
522    content,
523    ts.ScriptTarget.Latest,
524    true,
525    ts.ScriptKind.TS,
526  );
527
528  function visit(node: ts.Node): void {
529    if (TYPE_KEYWORD_KINDS.has(node.kind)) {
530      setSemanticForNode(semantics, sourceFile, node, PRIMITIVE_TYPE_INFO);
531    }
532
533    switch (node.kind) {
534      case ts.SyntaxKind.FalseKeyword:
535      case ts.SyntaxKind.TrueKeyword:
536        setSemanticForNode(semantics, sourceFile, node, BOOLEAN_INFO);
537        break;
538      case ts.SyntaxKind.NullKeyword:
539        setSemanticForNode(semantics, sourceFile, node, NULL_INFO);
540        break;
541    }
542
543    if (ts.isDecorator(node)) {
544      markDecoratorTarget(semantics, sourceFile, node.expression);
545    } else if (ts.isTypeAliasDeclaration(node)) {
546      setSemanticForNode(semantics, sourceFile, node.name, TYPE_ALIAS_INFO);
547    } else if (ts.isInterfaceDeclaration(node)) {
548      setSemanticForNode(semantics, sourceFile, node.name, INTERFACE_INFO);
549    } else if (ts.isClassDeclaration(node)) {
550      setSemanticForNode(semantics, sourceFile, node.name, CLASS_INFO);
551    } else if (ts.isEnumDeclaration(node)) {
552      setSemanticForNode(semantics, sourceFile, node.name, ENUM_INFO);
553    } else if (ts.isModuleDeclaration(node)) {
554      setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
555    } else if (ts.isTypeParameterDeclaration(node)) {
556      setSemanticForNode(semantics, sourceFile, node.name, TYPE_PARAMETER_INFO);
557    } else if (
558      ts.isFunctionDeclaration(node) ||
559      ts.isFunctionExpression(node)
560    ) {
561      setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO);
562    } else if (ts.isMethodDeclaration(node) || ts.isMethodSignature(node)) {
563      markPropertyName(semantics, sourceFile, node.name, FUNCTION_INFO);
564    } else if (
565      ts.isGetAccessorDeclaration(node) ||
566      ts.isSetAccessorDeclaration(node)
567    ) {
568      markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
569    } else if (ts.isPropertyDeclaration(node) || ts.isPropertySignature(node)) {
570      markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
571    } else if (ts.isConstructorDeclaration(node)) {
572      setSemanticForNode(
573        semantics,
574        sourceFile,
575        node.getFirstToken(sourceFile),
576        FUNCTION_INFO,
577      );
578    } else if (ts.isParameter(node)) {
579      if (hasParameterPropertyModifier(node)) {
580        markBindingName(semantics, sourceFile, node.name, PROPERTY_INFO);
581      } else {
582        markBindingName(semantics, sourceFile, node.name, PARAMETER_INFO);
583      }
584    } else if (ts.isVariableDeclaration(node)) {
585      if (
586        ts.isIdentifier(node.name) &&
587        node.initializer &&
588        (ts.isArrowFunction(node.initializer) ||
589          ts.isFunctionExpression(node.initializer))
590      ) {
591        setSemanticForNode(semantics, sourceFile, node.name, FUNCTION_INFO);
592      }
593    } else if (ts.isPropertyAssignment(node)) {
594      markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
595    } else if (ts.isEnumMember(node)) {
596      markPropertyName(semantics, sourceFile, node.name, ENUM_MEMBER_INFO);
597    } else if (ts.isPropertyAccessExpression(node)) {
598      markPropertyName(semantics, sourceFile, node.name, PROPERTY_INFO);
599    } else if (ts.isCallExpression(node)) {
600      markCallTarget(semantics, sourceFile, node.expression);
601    } else if (ts.isTaggedTemplateExpression(node)) {
602      markCallTarget(semantics, sourceFile, node.tag);
603    } else if (ts.isNewExpression(node)) {
604      markNewTarget(semantics, sourceFile, node.expression);
605    } else if (ts.isTypeReferenceNode(node)) {
606      markTypeName(semantics, sourceFile, node.typeName);
607    } else if (ts.isExpressionWithTypeArguments(node)) {
608      markTypeName(semantics, sourceFile, node.expression);
609    } else if (ts.isImportTypeNode(node) && node.qualifier) {
610      markTypeName(semantics, sourceFile, node.qualifier);
611    } else if (ts.isImportClause(node) && node.isTypeOnly && node.name) {
612      setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
613    } else if (ts.isImportSpecifier(node) && node.isTypeOnly) {
614      setSemanticForNode(
615        semantics,
616        sourceFile,
617        node.propertyName ?? node.name,
618        TYPE_INFO,
619      );
620      setSemanticForNode(semantics, sourceFile, node.name, TYPE_INFO);
621    }
622
623    ts.forEachChild(node, visit);
624  }
625
626  visit(sourceFile);
627  return semantics;
628}
629
630function applySemantics(
631  tokens: readonly ClewToken[],
632  semantics: Map<string, SemanticInfo>,
633): ClewToken[] {
634  if (semantics.size === 0) {
635    return [...tokens];
636  }
637
638  return tokens.map((token) => {
639    const semantic = semantics.get(semanticKey(token.start, token.end));
640    if (!semantic) {
641      return token;
642    }
643
644    return createToken(
645      token.text,
646      token.start,
647      token.end,
648      semantic.type,
649      semantic.scopes,
650      semantic.path,
651    );
652  });
653}
654
655function tokenizeTypescript(content: string): ClewOutput {
656  const baseTokens = baseTokensFromTypescript(content);
657  const semantics = collectTypescriptSemantics(content);
658  return { tokens: applySemantics(baseTokens, semantics) };
659}
660
661export const typescriptLanguageSupport: ClewLanguageSupport = {
662  ids: ["typescript", "ts", "mts", "cts", "javascript", "js", "mjs", "cjs"],
663  stableBoundary: stableLineBoundary,
664  tokenize: tokenizeTypescript,
665};