70160
Ce
cel-tui / files
cel-tui / packages / clew / src / languages / python.ts
12.9 KB
1import type { ClewOutput, ClewToken } from "../model.js"; 2import type { ClewLanguageSupport } from "../registry.js"; 3import { pushTokenRange, stableLineBoundary } from "../shared.js"; 4 5const PYTHON_KEYWORDS = new Set([ 6 "and", 7 "as", 8 "assert", 9 "async", 10 "await", 11 "break", 12 "case", 13 "class", 14 "continue", 15 "def", 16 "del", 17 "elif", 18 "else", 19 "except", 20 "finally", 21 "for", 22 "from", 23 "global", 24 "if", 25 "import", 26 "in", 27 "is", 28 "lambda", 29 "match", 30 "nonlocal", 31 "not", 32 "or", 33 "pass", 34 "raise", 35 "return", 36 "try", 37 "while", 38 "with", 39 "yield", 40]); 41 42const PYTHON_BUILTINS = new Set([ 43 "abs", 44 "aiter", 45 "all", 46 "anext", 47 "any", 48 "ascii", 49 "bin", 50 "breakpoint", 51 "callable", 52 "chr", 53 "classmethod", 54 "compile", 55 "delattr", 56 "dir", 57 "divmod", 58 "enumerate", 59 "eval", 60 "exec", 61 "filter", 62 "format", 63 "getattr", 64 "globals", 65 "hasattr", 66 "hash", 67 "help", 68 "hex", 69 "id", 70 "input", 71 "isinstance", 72 "issubclass", 73 "iter", 74 "len", 75 "locals", 76 "map", 77 "max", 78 "min", 79 "next", 80 "oct", 81 "open", 82 "ord", 83 "pow", 84 "print", 85 "property", 86 "repr", 87 "reversed", 88 "round", 89 "setattr", 90 "sorted", 91 "staticmethod", 92 "sum", 93 "super", 94 "vars", 95 "zip", 96 "__import__", 97]); 98 99const PYTHON_TYPE_NAMES = new Set([ 100 "BaseException", 101 "Exception", 102 "bool", 103 "bytearray", 104 "bytes", 105 "complex", 106 "dict", 107 "float", 108 "frozenset", 109 "int", 110 "list", 111 "memoryview", 112 "object", 113 "range", 114 "set", 115 "slice", 116 "str", 117 "tuple", 118 "type", 119]); 120 121const STRING_PREFIX_CHARS = new Set(["b", "f", "r", "u"]); 122 123const OPERATORS = [ 124 "**=", 125 "//=", 126 "<<=", 127 ">>=", 128 "@=", 129 ":=", 130 "==", 131 "!=", 132 "<=", 133 ">=", 134 "->", 135 "+=", 136 "-=", 137 "*=", 138 "/=", 139 "%=", 140 "&=", 141 "|=", 142 "^=", 143 "**", 144 "//", 145 "<<", 146 ">>", 147 "=", 148 "+", 149 "-", 150 "*", 151 "/", 152 "%", 153 "@", 154 "&", 155 "|", 156 "^", 157 "~", 158 "<", 159 ">", 160] as const; 161 162interface PythonStringStart { 163 prefixEnd: number; 164 quote: '"' | "'"; 165 raw: boolean; 166 triple: boolean; 167} 168 169interface PythonTokenizerState { 170 content: string; 171 expectIdentifierType: "function" | "type" | null; 172 index: number; 173 lastSignificantToken: ClewToken | null; 174 tokens: ClewToken[]; 175} 176 177function pushToken( 178 state: PythonTokenizerState, 179 start: number, 180 end: number, 181 type: string, 182 scopes?: readonly string[], 183): void { 184 pushTokenRange(state.tokens, state.content, start, end, type, scopes); 185 186 const token = state.tokens.at(-1); 187 if (!token) { 188 return; 189 } 190 191 if (type !== "comment" && type !== "whitespace") { 192 state.lastSignificantToken = token; 193 } 194} 195 196function isWhitespace(char: string | undefined): boolean { 197 return char !== undefined && /\s/.test(char); 198} 199 200function isIdentifierStart(char: string | undefined): boolean { 201 return char !== undefined && /[A-Za-z_]/.test(char); 202} 203 204function isIdentifierPart(char: string | undefined): boolean { 205 return char !== undefined && /[A-Za-z0-9_]/.test(char); 206} 207 208function isStringPrefixChar(char: string | undefined): boolean { 209 return char !== undefined && STRING_PREFIX_CHARS.has(char.toLowerCase()); 210} 211 212function isDigit(char: string | undefined): boolean { 213 return char !== undefined && char >= "0" && char <= "9"; 214} 215 216function isBinaryDigit(char: string | undefined): boolean { 217 return char === "0" || char === "1"; 218} 219 220function isOctalDigit(char: string | undefined): boolean { 221 return char !== undefined && char >= "0" && char <= "7"; 222} 223 224function isHexDigit(char: string | undefined): boolean { 225 return char !== undefined && /[0-9a-fA-F]/.test(char); 226} 227 228function readDigitsWithUnderscores( 229 content: string, 230 start: number, 231 predicate: (char: string | undefined) => boolean, 232): number { 233 let index = start; 234 let sawDigit = false; 235 236 while (true) { 237 const char = content[index]; 238 if (predicate(char)) { 239 sawDigit = true; 240 index += 1; 241 continue; 242 } 243 244 if (char === "_" && sawDigit && predicate(content[index + 1])) { 245 index += 1; 246 continue; 247 } 248 249 break; 250 } 251 252 return sawDigit ? index : -1; 253} 254 255function readExponentPart(content: string, start: number): number { 256 const marker = content[start]; 257 if (marker !== "e" && marker !== "E") { 258 return start; 259 } 260 261 let index = start + 1; 262 if (content[index] === "+" || content[index] === "-") { 263 index += 1; 264 } 265 266 const digitsEnd = readDigitsWithUnderscores(content, index, isDigit); 267 return digitsEnd === -1 ? start : digitsEnd; 268} 269 270function readNumberEnd(content: string, start: number): number { 271 if (content[start] === ".") { 272 const fractionEnd = readDigitsWithUnderscores(content, start + 1, isDigit); 273 if (fractionEnd === -1) { 274 return start; 275 } 276 277 let index = readExponentPart(content, fractionEnd); 278 if (content[index] === "j" || content[index] === "J") { 279 index += 1; 280 } 281 return index; 282 } 283 284 if (content[start] === "0") { 285 const marker = content[start + 1]; 286 if (marker === "b" || marker === "B") { 287 const end = readDigitsWithUnderscores(content, start + 2, isBinaryDigit); 288 return end === -1 ? Math.min(content.length, start + 2) : end; 289 } 290 if (marker === "o" || marker === "O") { 291 const end = readDigitsWithUnderscores(content, start + 2, isOctalDigit); 292 return end === -1 ? Math.min(content.length, start + 2) : end; 293 } 294 if (marker === "x" || marker === "X") { 295 const end = readDigitsWithUnderscores(content, start + 2, isHexDigit); 296 return end === -1 ? Math.min(content.length, start + 2) : end; 297 } 298 } 299 300 let index = readDigitsWithUnderscores(content, start, isDigit); 301 if (index === -1) { 302 return start; 303 } 304 305 if (content[index] === "." && content[index + 1] !== ".") { 306 index += 1; 307 const fractionEnd = readDigitsWithUnderscores(content, index, isDigit); 308 if (fractionEnd !== -1) { 309 index = fractionEnd; 310 } 311 } 312 313 index = readExponentPart(content, index); 314 if (content[index] === "j" || content[index] === "J") { 315 index += 1; 316 } 317 318 return index; 319} 320 321function readIdentifierEnd(content: string, start: number): number { 322 let index = start + 1; 323 while (isIdentifierPart(content[index])) { 324 index += 1; 325 } 326 return index; 327} 328 329function readStringStart( 330 content: string, 331 start: number, 332): PythonStringStart | undefined { 333 const first = content[start]; 334 if (first === '"' || first === "'") { 335 return { 336 prefixEnd: start, 337 quote: first, 338 raw: false, 339 triple: content.startsWith(first.repeat(3), start), 340 }; 341 } 342 343 if (!isStringPrefixChar(first)) { 344 return undefined; 345 } 346 347 let prefixEnd = start; 348 while ( 349 prefixEnd < content.length && 350 prefixEnd - start < 2 && 351 isStringPrefixChar(content[prefixEnd]) 352 ) { 353 prefixEnd += 1; 354 } 355 356 const quote = content[prefixEnd]; 357 if (quote !== '"' && quote !== "'") { 358 return undefined; 359 } 360 361 return { 362 prefixEnd, 363 quote, 364 raw: /r/i.test(content.slice(start, prefixEnd)), 365 triple: content.startsWith(quote.repeat(3), prefixEnd), 366 }; 367} 368 369function readStringEnd( 370 content: string, 371 _start: number, 372 stringStart: PythonStringStart, 373): number { 374 const closing = stringStart.triple 375 ? stringStart.quote.repeat(3) 376 : stringStart.quote; 377 let index = stringStart.prefixEnd + closing.length; 378 379 while (index < content.length) { 380 if (!stringStart.raw && content[index] === "\\") { 381 index += 2; 382 continue; 383 } 384 385 if (content.startsWith(closing, index)) { 386 return index + closing.length; 387 } 388 389 index += 1; 390 } 391 392 return content.length; 393} 394 395function readCommentEnd(content: string, start: number): number { 396 let index = start; 397 while ( 398 index < content.length && 399 content[index] !== "\n" && 400 content[index] !== "\r" 401 ) { 402 index += 1; 403 } 404 return index; 405} 406 407function readWhitespaceEnd(content: string, start: number): number { 408 let index = start; 409 while (isWhitespace(content[index])) { 410 index += 1; 411 } 412 return index; 413} 414 415function isDecoratorStart(content: string, start: number): boolean { 416 if (content[start] !== "@" || !isIdentifierStart(content[start + 1])) { 417 return false; 418 } 419 420 let index = start - 1; 421 while (index >= 0 && (content[index] === " " || content[index] === "\t")) { 422 index -= 1; 423 } 424 425 return index < 0 || content[index] === "\n" || content[index] === "\r"; 426} 427 428function readDecoratorEnd(content: string, start: number): number { 429 let index = start + 1; 430 431 while (true) { 432 if (!isIdentifierStart(content[index])) { 433 return index; 434 } 435 436 index = readIdentifierEnd(content, index); 437 if (content[index] !== "." || !isIdentifierStart(content[index + 1])) { 438 return index; 439 } 440 441 index += 1; 442 } 443} 444 445function matchOperator(content: string, start: number): string | undefined { 446 return OPERATORS.find((operator) => content.startsWith(operator, start)); 447} 448 449function readIdentifier(state: PythonTokenizerState): void { 450 const start = state.index; 451 const end = readIdentifierEnd(state.content, start); 452 const text = state.content.slice(start, end); 453 454 if (text === "True" || text === "False") { 455 pushToken(state, start, end, "keyword", ["keyword", "constant.boolean"]); 456 state.expectIdentifierType = null; 457 state.index = end; 458 return; 459 } 460 461 if (text === "None") { 462 pushToken(state, start, end, "keyword", ["keyword", "constant.none"]); 463 state.expectIdentifierType = null; 464 state.index = end; 465 return; 466 } 467 468 if (PYTHON_KEYWORDS.has(text)) { 469 pushToken(state, start, end, "keyword"); 470 state.expectIdentifierType = 471 text === "def" ? "function" : text === "class" ? "type" : null; 472 state.index = end; 473 return; 474 } 475 476 if (state.expectIdentifierType) { 477 pushToken(state, start, end, state.expectIdentifierType); 478 state.expectIdentifierType = null; 479 state.index = end; 480 return; 481 } 482 483 if (state.lastSignificantToken?.text === ".") { 484 pushToken(state, start, end, "property"); 485 state.index = end; 486 return; 487 } 488 489 if (PYTHON_TYPE_NAMES.has(text)) { 490 pushToken(state, start, end, "type"); 491 state.index = end; 492 return; 493 } 494 495 if (PYTHON_BUILTINS.has(text)) { 496 pushToken(state, start, end, "builtin"); 497 state.index = end; 498 return; 499 } 500 501 pushToken(state, start, end, "identifier", ["identifier", "variable"]); 502 state.index = end; 503} 504 505function tokenizePython(content: string): ClewOutput { 506 const state: PythonTokenizerState = { 507 content, 508 expectIdentifierType: null, 509 index: 0, 510 lastSignificantToken: null, 511 tokens: [], 512 }; 513 514 while (state.index < content.length) { 515 const char = content[state.index]; 516 if (char === undefined) { 517 break; 518 } 519 520 if (isWhitespace(char)) { 521 const end = readWhitespaceEnd(content, state.index); 522 const whitespace = content.slice(state.index, end); 523 pushToken(state, state.index, end, "whitespace"); 524 if (whitespace.includes("\n") || whitespace.includes("\r")) { 525 state.expectIdentifierType = null; 526 state.lastSignificantToken = null; 527 } 528 state.index = end; 529 continue; 530 } 531 532 if (char === "#") { 533 const end = readCommentEnd(content, state.index); 534 pushToken(state, state.index, end, "comment"); 535 state.index = end; 536 continue; 537 } 538 539 if (isDecoratorStart(content, state.index)) { 540 const end = readDecoratorEnd(content, state.index); 541 pushToken(state, state.index, end, "meta", ["meta", "meta.decorator"]); 542 state.expectIdentifierType = null; 543 state.index = end; 544 continue; 545 } 546 547 const stringStart = readStringStart(content, state.index); 548 if (stringStart) { 549 const end = readStringEnd(content, state.index, stringStart); 550 pushToken(state, state.index, end, "string"); 551 state.expectIdentifierType = null; 552 state.index = end; 553 continue; 554 } 555 556 if (char === "." && isDigit(content[state.index + 1])) { 557 const end = readNumberEnd(content, state.index); 558 pushToken(state, state.index, end, "number"); 559 state.expectIdentifierType = null; 560 state.index = end; 561 continue; 562 } 563 564 if (isDigit(char)) { 565 const end = readNumberEnd(content, state.index); 566 pushToken(state, state.index, end, "number"); 567 state.expectIdentifierType = null; 568 state.index = end; 569 continue; 570 } 571 572 if (isIdentifierStart(char)) { 573 readIdentifier(state); 574 continue; 575 } 576 577 if (content.startsWith("...", state.index)) { 578 pushToken(state, state.index, state.index + 3, "punctuation"); 579 state.expectIdentifierType = null; 580 state.index += 3; 581 continue; 582 } 583 584 const operator = matchOperator(content, state.index); 585 if (operator) { 586 pushToken(state, state.index, state.index + operator.length, "operator"); 587 state.expectIdentifierType = null; 588 state.index += operator.length; 589 continue; 590 } 591 592 if ("()[]{}.,:;".includes(char)) { 593 pushToken(state, state.index, state.index + 1, "punctuation"); 594 state.expectIdentifierType = null; 595 state.index += 1; 596 continue; 597 } 598 599 pushToken(state, state.index, state.index + 1, "text"); 600 state.expectIdentifierType = null; 601 state.index += 1; 602 } 603 604 return { tokens: state.tokens }; 605} 606 607export const pythonLanguageSupport: ClewLanguageSupport = { 608 ids: ["python", "py"], 609 stableBoundary: stableLineBoundary, 610 tokenize: tokenizePython, 611};