xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / languages / python.ts

12.9 KB
1import type { ClewOutput, ClewToken } from "../model.js";
2import type { ClewLanguageSupport } from "../registry.js";
3import { pushTokenRange, stableLineBoundary } from "../shared.js";
4
5const PYTHON_KEYWORDS = new Set([
6  "and",
7  "as",
8  "assert",
9  "async",
10  "await",
11  "break",
12  "case",
13  "class",
14  "continue",
15  "def",
16  "del",
17  "elif",
18  "else",
19  "except",
20  "finally",
21  "for",
22  "from",
23  "global",
24  "if",
25  "import",
26  "in",
27  "is",
28  "lambda",
29  "match",
30  "nonlocal",
31  "not",
32  "or",
33  "pass",
34  "raise",
35  "return",
36  "try",
37  "while",
38  "with",
39  "yield",
40]);
41
42const PYTHON_BUILTINS = new Set([
43  "abs",
44  "aiter",
45  "all",
46  "anext",
47  "any",
48  "ascii",
49  "bin",
50  "breakpoint",
51  "callable",
52  "chr",
53  "classmethod",
54  "compile",
55  "delattr",
56  "dir",
57  "divmod",
58  "enumerate",
59  "eval",
60  "exec",
61  "filter",
62  "format",
63  "getattr",
64  "globals",
65  "hasattr",
66  "hash",
67  "help",
68  "hex",
69  "id",
70  "input",
71  "isinstance",
72  "issubclass",
73  "iter",
74  "len",
75  "locals",
76  "map",
77  "max",
78  "min",
79  "next",
80  "oct",
81  "open",
82  "ord",
83  "pow",
84  "print",
85  "property",
86  "repr",
87  "reversed",
88  "round",
89  "setattr",
90  "sorted",
91  "staticmethod",
92  "sum",
93  "super",
94  "vars",
95  "zip",
96  "__import__",
97]);
98
99const PYTHON_TYPE_NAMES = new Set([
100  "BaseException",
101  "Exception",
102  "bool",
103  "bytearray",
104  "bytes",
105  "complex",
106  "dict",
107  "float",
108  "frozenset",
109  "int",
110  "list",
111  "memoryview",
112  "object",
113  "range",
114  "set",
115  "slice",
116  "str",
117  "tuple",
118  "type",
119]);
120
121const STRING_PREFIX_CHARS = new Set(["b", "f", "r", "u"]);
122
123const OPERATORS = [
124  "**=",
125  "//=",
126  "<<=",
127  ">>=",
128  "@=",
129  ":=",
130  "==",
131  "!=",
132  "<=",
133  ">=",
134  "->",
135  "+=",
136  "-=",
137  "*=",
138  "/=",
139  "%=",
140  "&=",
141  "|=",
142  "^=",
143  "**",
144  "//",
145  "<<",
146  ">>",
147  "=",
148  "+",
149  "-",
150  "*",
151  "/",
152  "%",
153  "@",
154  "&",
155  "|",
156  "^",
157  "~",
158  "<",
159  ">",
160] as const;
161
162interface PythonStringStart {
163  prefixEnd: number;
164  quote: '"' | "'";
165  raw: boolean;
166  triple: boolean;
167}
168
169interface PythonTokenizerState {
170  content: string;
171  expectIdentifierType: "function" | "type" | null;
172  index: number;
173  lastSignificantToken: ClewToken | null;
174  tokens: ClewToken[];
175}
176
177function pushToken(
178  state: PythonTokenizerState,
179  start: number,
180  end: number,
181  type: string,
182  scopes?: readonly string[],
183): void {
184  pushTokenRange(state.tokens, state.content, start, end, type, scopes);
185
186  const token = state.tokens.at(-1);
187  if (!token) {
188    return;
189  }
190
191  if (type !== "comment" && type !== "whitespace") {
192    state.lastSignificantToken = token;
193  }
194}
195
196function isWhitespace(char: string | undefined): boolean {
197  return char !== undefined && /\s/.test(char);
198}
199
200function isIdentifierStart(char: string | undefined): boolean {
201  return char !== undefined && /[A-Za-z_]/.test(char);
202}
203
204function isIdentifierPart(char: string | undefined): boolean {
205  return char !== undefined && /[A-Za-z0-9_]/.test(char);
206}
207
208function isStringPrefixChar(char: string | undefined): boolean {
209  return char !== undefined && STRING_PREFIX_CHARS.has(char.toLowerCase());
210}
211
212function isDigit(char: string | undefined): boolean {
213  return char !== undefined && char >= "0" && char <= "9";
214}
215
216function isBinaryDigit(char: string | undefined): boolean {
217  return char === "0" || char === "1";
218}
219
220function isOctalDigit(char: string | undefined): boolean {
221  return char !== undefined && char >= "0" && char <= "7";
222}
223
224function isHexDigit(char: string | undefined): boolean {
225  return char !== undefined && /[0-9a-fA-F]/.test(char);
226}
227
228function readDigitsWithUnderscores(
229  content: string,
230  start: number,
231  predicate: (char: string | undefined) => boolean,
232): number {
233  let index = start;
234  let sawDigit = false;
235
236  while (true) {
237    const char = content[index];
238    if (predicate(char)) {
239      sawDigit = true;
240      index += 1;
241      continue;
242    }
243
244    if (char === "_" && sawDigit && predicate(content[index + 1])) {
245      index += 1;
246      continue;
247    }
248
249    break;
250  }
251
252  return sawDigit ? index : -1;
253}
254
255function readExponentPart(content: string, start: number): number {
256  const marker = content[start];
257  if (marker !== "e" && marker !== "E") {
258    return start;
259  }
260
261  let index = start + 1;
262  if (content[index] === "+" || content[index] === "-") {
263    index += 1;
264  }
265
266  const digitsEnd = readDigitsWithUnderscores(content, index, isDigit);
267  return digitsEnd === -1 ? start : digitsEnd;
268}
269
270function readNumberEnd(content: string, start: number): number {
271  if (content[start] === ".") {
272    const fractionEnd = readDigitsWithUnderscores(content, start + 1, isDigit);
273    if (fractionEnd === -1) {
274      return start;
275    }
276
277    let index = readExponentPart(content, fractionEnd);
278    if (content[index] === "j" || content[index] === "J") {
279      index += 1;
280    }
281    return index;
282  }
283
284  if (content[start] === "0") {
285    const marker = content[start + 1];
286    if (marker === "b" || marker === "B") {
287      const end = readDigitsWithUnderscores(content, start + 2, isBinaryDigit);
288      return end === -1 ? Math.min(content.length, start + 2) : end;
289    }
290    if (marker === "o" || marker === "O") {
291      const end = readDigitsWithUnderscores(content, start + 2, isOctalDigit);
292      return end === -1 ? Math.min(content.length, start + 2) : end;
293    }
294    if (marker === "x" || marker === "X") {
295      const end = readDigitsWithUnderscores(content, start + 2, isHexDigit);
296      return end === -1 ? Math.min(content.length, start + 2) : end;
297    }
298  }
299
300  let index = readDigitsWithUnderscores(content, start, isDigit);
301  if (index === -1) {
302    return start;
303  }
304
305  if (content[index] === "." && content[index + 1] !== ".") {
306    index += 1;
307    const fractionEnd = readDigitsWithUnderscores(content, index, isDigit);
308    if (fractionEnd !== -1) {
309      index = fractionEnd;
310    }
311  }
312
313  index = readExponentPart(content, index);
314  if (content[index] === "j" || content[index] === "J") {
315    index += 1;
316  }
317
318  return index;
319}
320
321function readIdentifierEnd(content: string, start: number): number {
322  let index = start + 1;
323  while (isIdentifierPart(content[index])) {
324    index += 1;
325  }
326  return index;
327}
328
329function readStringStart(
330  content: string,
331  start: number,
332): PythonStringStart | undefined {
333  const first = content[start];
334  if (first === '"' || first === "'") {
335    return {
336      prefixEnd: start,
337      quote: first,
338      raw: false,
339      triple: content.startsWith(first.repeat(3), start),
340    };
341  }
342
343  if (!isStringPrefixChar(first)) {
344    return undefined;
345  }
346
347  let prefixEnd = start;
348  while (
349    prefixEnd < content.length &&
350    prefixEnd - start < 2 &&
351    isStringPrefixChar(content[prefixEnd])
352  ) {
353    prefixEnd += 1;
354  }
355
356  const quote = content[prefixEnd];
357  if (quote !== '"' && quote !== "'") {
358    return undefined;
359  }
360
361  return {
362    prefixEnd,
363    quote,
364    raw: /r/i.test(content.slice(start, prefixEnd)),
365    triple: content.startsWith(quote.repeat(3), prefixEnd),
366  };
367}
368
369function readStringEnd(
370  content: string,
371  _start: number,
372  stringStart: PythonStringStart,
373): number {
374  const closing = stringStart.triple
375    ? stringStart.quote.repeat(3)
376    : stringStart.quote;
377  let index = stringStart.prefixEnd + closing.length;
378
379  while (index < content.length) {
380    if (!stringStart.raw && content[index] === "\\") {
381      index += 2;
382      continue;
383    }
384
385    if (content.startsWith(closing, index)) {
386      return index + closing.length;
387    }
388
389    index += 1;
390  }
391
392  return content.length;
393}
394
395function readCommentEnd(content: string, start: number): number {
396  let index = start;
397  while (
398    index < content.length &&
399    content[index] !== "\n" &&
400    content[index] !== "\r"
401  ) {
402    index += 1;
403  }
404  return index;
405}
406
407function readWhitespaceEnd(content: string, start: number): number {
408  let index = start;
409  while (isWhitespace(content[index])) {
410    index += 1;
411  }
412  return index;
413}
414
415function isDecoratorStart(content: string, start: number): boolean {
416  if (content[start] !== "@" || !isIdentifierStart(content[start + 1])) {
417    return false;
418  }
419
420  let index = start - 1;
421  while (index >= 0 && (content[index] === " " || content[index] === "\t")) {
422    index -= 1;
423  }
424
425  return index < 0 || content[index] === "\n" || content[index] === "\r";
426}
427
428function readDecoratorEnd(content: string, start: number): number {
429  let index = start + 1;
430
431  while (true) {
432    if (!isIdentifierStart(content[index])) {
433      return index;
434    }
435
436    index = readIdentifierEnd(content, index);
437    if (content[index] !== "." || !isIdentifierStart(content[index + 1])) {
438      return index;
439    }
440
441    index += 1;
442  }
443}
444
445function matchOperator(content: string, start: number): string | undefined {
446  return OPERATORS.find((operator) => content.startsWith(operator, start));
447}
448
449function readIdentifier(state: PythonTokenizerState): void {
450  const start = state.index;
451  const end = readIdentifierEnd(state.content, start);
452  const text = state.content.slice(start, end);
453
454  if (text === "True" || text === "False") {
455    pushToken(state, start, end, "keyword", ["keyword", "constant.boolean"]);
456    state.expectIdentifierType = null;
457    state.index = end;
458    return;
459  }
460
461  if (text === "None") {
462    pushToken(state, start, end, "keyword", ["keyword", "constant.none"]);
463    state.expectIdentifierType = null;
464    state.index = end;
465    return;
466  }
467
468  if (PYTHON_KEYWORDS.has(text)) {
469    pushToken(state, start, end, "keyword");
470    state.expectIdentifierType =
471      text === "def" ? "function" : text === "class" ? "type" : null;
472    state.index = end;
473    return;
474  }
475
476  if (state.expectIdentifierType) {
477    pushToken(state, start, end, state.expectIdentifierType);
478    state.expectIdentifierType = null;
479    state.index = end;
480    return;
481  }
482
483  if (state.lastSignificantToken?.text === ".") {
484    pushToken(state, start, end, "property");
485    state.index = end;
486    return;
487  }
488
489  if (PYTHON_TYPE_NAMES.has(text)) {
490    pushToken(state, start, end, "type");
491    state.index = end;
492    return;
493  }
494
495  if (PYTHON_BUILTINS.has(text)) {
496    pushToken(state, start, end, "builtin");
497    state.index = end;
498    return;
499  }
500
501  pushToken(state, start, end, "identifier", ["identifier", "variable"]);
502  state.index = end;
503}
504
505function tokenizePython(content: string): ClewOutput {
506  const state: PythonTokenizerState = {
507    content,
508    expectIdentifierType: null,
509    index: 0,
510    lastSignificantToken: null,
511    tokens: [],
512  };
513
514  while (state.index < content.length) {
515    const char = content[state.index];
516    if (char === undefined) {
517      break;
518    }
519
520    if (isWhitespace(char)) {
521      const end = readWhitespaceEnd(content, state.index);
522      const whitespace = content.slice(state.index, end);
523      pushToken(state, state.index, end, "whitespace");
524      if (whitespace.includes("\n") || whitespace.includes("\r")) {
525        state.expectIdentifierType = null;
526        state.lastSignificantToken = null;
527      }
528      state.index = end;
529      continue;
530    }
531
532    if (char === "#") {
533      const end = readCommentEnd(content, state.index);
534      pushToken(state, state.index, end, "comment");
535      state.index = end;
536      continue;
537    }
538
539    if (isDecoratorStart(content, state.index)) {
540      const end = readDecoratorEnd(content, state.index);
541      pushToken(state, state.index, end, "meta", ["meta", "meta.decorator"]);
542      state.expectIdentifierType = null;
543      state.index = end;
544      continue;
545    }
546
547    const stringStart = readStringStart(content, state.index);
548    if (stringStart) {
549      const end = readStringEnd(content, state.index, stringStart);
550      pushToken(state, state.index, end, "string");
551      state.expectIdentifierType = null;
552      state.index = end;
553      continue;
554    }
555
556    if (char === "." && isDigit(content[state.index + 1])) {
557      const end = readNumberEnd(content, state.index);
558      pushToken(state, state.index, end, "number");
559      state.expectIdentifierType = null;
560      state.index = end;
561      continue;
562    }
563
564    if (isDigit(char)) {
565      const end = readNumberEnd(content, state.index);
566      pushToken(state, state.index, end, "number");
567      state.expectIdentifierType = null;
568      state.index = end;
569      continue;
570    }
571
572    if (isIdentifierStart(char)) {
573      readIdentifier(state);
574      continue;
575    }
576
577    if (content.startsWith("...", state.index)) {
578      pushToken(state, state.index, state.index + 3, "punctuation");
579      state.expectIdentifierType = null;
580      state.index += 3;
581      continue;
582    }
583
584    const operator = matchOperator(content, state.index);
585    if (operator) {
586      pushToken(state, state.index, state.index + operator.length, "operator");
587      state.expectIdentifierType = null;
588      state.index += operator.length;
589      continue;
590    }
591
592    if ("()[]{}.,:;".includes(char)) {
593      pushToken(state, state.index, state.index + 1, "punctuation");
594      state.expectIdentifierType = null;
595      state.index += 1;
596      continue;
597    }
598
599    pushToken(state, state.index, state.index + 1, "text");
600    state.expectIdentifierType = null;
601    state.index += 1;
602  }
603
604  return { tokens: state.tokens };
605}
606
607export const pythonLanguageSupport: ClewLanguageSupport = {
608  ids: ["python", "py"],
609  stableBoundary: stableLineBoundary,
610  tokenize: tokenizePython,
611};