70160
Ce
cel-tui / files
cel-tui / packages / clew / src / languages / bash.ts
18.7 KB
1import type { ClewOutput, ClewToken } from "../model.js"; 2import type { ClewLanguageSupport } from "../registry.js"; 3import { pushTokenRange, stableLineBoundary } from "../shared.js"; 4 5const BASH_KEYWORDS = new Set([ 6 "if", 7 "then", 8 "elif", 9 "else", 10 "fi", 11 "for", 12 "while", 13 "until", 14 "do", 15 "done", 16 "case", 17 "esac", 18 "select", 19 "function", 20 "in", 21 "time", 22 "coproc", 23]); 24 25const COMMAND_FOLLOWING_KEYWORDS = new Set([ 26 "if", 27 "then", 28 "elif", 29 "else", 30 "while", 31 "until", 32 "do", 33 "time", 34 "coproc", 35]); 36 37const BASH_BUILTINS = new Set([ 38 ".", 39 ":", 40 "alias", 41 "bg", 42 "bind", 43 "break", 44 "builtin", 45 "caller", 46 "cd", 47 "command", 48 "compgen", 49 "complete", 50 "compopt", 51 "continue", 52 "declare", 53 "dirs", 54 "disown", 55 "echo", 56 "enable", 57 "eval", 58 "exec", 59 "exit", 60 "export", 61 "false", 62 "fc", 63 "fg", 64 "getopts", 65 "hash", 66 "help", 67 "history", 68 "jobs", 69 "kill", 70 "let", 71 "local", 72 "logout", 73 "mapfile", 74 "popd", 75 "printf", 76 "pushd", 77 "pwd", 78 "read", 79 "readarray", 80 "readonly", 81 "return", 82 "set", 83 "shift", 84 "shopt", 85 "source", 86 "suspend", 87 "test", 88 "times", 89 "trap", 90 "true", 91 "type", 92 "typeset", 93 "ulimit", 94 "umask", 95 "unalias", 96 "unset", 97 "wait", 98]); 99 100const COMMAND_BOUNDARY_OPERATORS = new Set([ 101 "&", 102 "&&", 103 "(", 104 ";", 105 ";;", 106 ";&", 107 ";;&", 108 "|", 109 "|&", 110 "||", 111 "{", 112]); 113 114const OPERATORS = [ 115 "<<-", 116 "<<<", 117 "<<", 118 ">>", 119 "||", 120 "&&", 121 "|&", 122 "&>>", 123 "&>", 124 ">&", 125 "<&", 126 "<>", 127 ">|", 128 ";;&", 129 ";&", 130 ";;", 131 "[[", 132 "]]", 133 "|", 134 "&", 135 ";", 136 "(", 137 ")", 138 "{", 139 "}", 140 "<", 141 ">", 142] as const; 143 144interface PendingHeredoc { 145 allowIndent: boolean; 146} 147 148interface HeredocSpec { 149 allowIndent: boolean; 150 delimiter: string; 151} 152 153interface BashTokenizerState { 154 assignmentPending: boolean; 155 canStartComment: boolean; 156 commandWordPending: boolean; 157 content: string; 158 expectCaseIn: boolean; 159 expectCaseSubject: boolean; 160 expectForIn: boolean; 161 expectForName: boolean; 162 expectFunctionName: boolean; 163 heredocQueue: HeredocSpec[]; 164 index: number; 165 pendingHeredoc: PendingHeredoc | null; 166 tokens: ClewToken[]; 167} 168 169function isSpace(char: string | undefined): boolean { 170 return char === " " || char === "\t"; 171} 172 173function isNewlineStart(content: string, index: number): boolean { 174 return content[index] === "\n" || content[index] === "\r"; 175} 176 177function isWordBoundaryStart(content: string, index: number): boolean { 178 const char = content[index]; 179 if (char === undefined) { 180 return true; 181 } 182 183 return ( 184 char === "$" || 185 char === '"' || 186 char === "'" || 187 char === "`" || 188 char === "\\" || 189 isSpace(char) || 190 isNewlineStart(content, index) || 191 OPERATORS.some((operator) => content.startsWith(operator, index)) 192 ); 193} 194 195function pushToken( 196 state: BashTokenizerState, 197 start: number, 198 end: number, 199 type: string, 200 scopes?: readonly string[], 201): void { 202 pushTokenRange(state.tokens, state.content, start, end, type, scopes); 203} 204 205function readWhile( 206 state: BashTokenizerState, 207 predicate: (char: string | undefined, index: number) => boolean, 208): number { 209 const start = state.index; 210 while (predicate(state.content[state.index], state.index)) { 211 state.index += 1; 212 } 213 return start; 214} 215 216function readLineEnd(content: string, index: number): number { 217 if (content[index] === "\r" && content[index + 1] === "\n") { 218 return index + 2; 219 } 220 return index + 1; 221} 222 223function matchOperator(content: string, index: number): string | undefined { 224 return OPERATORS.find((operator) => content.startsWith(operator, index)); 225} 226 227function readPlainSegmentEnd(content: string, index: number): number { 228 let end = index; 229 while (!isWordBoundaryStart(content, end)) { 230 end += 1; 231 } 232 return end; 233} 234 235function isAssignmentPrefix(text: string): boolean { 236 return /^[A-Za-z_][A-Za-z0-9_]*=$/.test(text); 237} 238 239function isNumber(text: string): boolean { 240 return /^(?:\d+|0x[0-9a-fA-F]+)$/.test(text); 241} 242 243function registerHeredoc( 244 state: BashTokenizerState, 245 rawDelimiter: string, 246): void { 247 if (!state.pendingHeredoc) { 248 return; 249 } 250 251 state.heredocQueue.push({ 252 allowIndent: state.pendingHeredoc.allowIndent, 253 delimiter: rawDelimiter, 254 }); 255 state.pendingHeredoc = null; 256} 257 258function classifyKeywordState(state: BashTokenizerState, text: string): void { 259 state.expectFunctionName = text === "function"; 260 state.expectForName = text === "for" || text === "select"; 261 state.expectCaseSubject = text === "case"; 262 263 if (text === "in") { 264 state.expectForIn = false; 265 state.expectCaseIn = false; 266 } else if (!state.expectForName) { 267 state.expectForIn = state.expectForIn && text !== "for"; 268 } 269 270 if (text !== "case") { 271 state.expectCaseIn = state.expectCaseIn && text !== "in"; 272 } 273 274 state.assignmentPending = false; 275 state.commandWordPending = COMMAND_FOLLOWING_KEYWORDS.has(text); 276} 277 278function pushKeyword( 279 state: BashTokenizerState, 280 start: number, 281 end: number, 282): void { 283 pushToken(state, start, end, "keyword"); 284 classifyKeywordState(state, state.content.slice(start, end)); 285} 286 287function pushCommandLike( 288 state: BashTokenizerState, 289 start: number, 290 end: number, 291 type: "builtin" | "command" | "function" | "number" | "text" | "variable", 292 scopes?: readonly string[], 293): void { 294 pushToken(state, start, end, type, scopes); 295 state.assignmentPending = false; 296 state.commandWordPending = false; 297} 298 299function readComment(state: BashTokenizerState): void { 300 const start = state.index; 301 while ( 302 state.index < state.content.length && 303 !isNewlineStart(state.content, state.index) 304 ) { 305 state.index += 1; 306 } 307 pushToken(state, start, state.index, "comment"); 308 state.assignmentPending = false; 309 state.canStartComment = false; 310 state.commandWordPending = false; 311} 312 313function readSpaces(state: BashTokenizerState): void { 314 const start = readWhile(state, (char) => isSpace(char)); 315 pushToken(state, start, state.index, "whitespace"); 316 state.assignmentPending = false; 317 state.canStartComment = true; 318} 319 320function consumeHeredocBodies(state: BashTokenizerState): void { 321 while (state.heredocQueue.length > 0 && state.index < state.content.length) { 322 const heredoc = state.heredocQueue[0]; 323 if (!heredoc) { 324 return; 325 } 326 327 const lineStart = state.index; 328 while ( 329 state.index < state.content.length && 330 !isNewlineStart(state.content, state.index) 331 ) { 332 state.index += 1; 333 } 334 335 const lineEnd = state.index; 336 const rawLine = state.content.slice(lineStart, lineEnd); 337 const comparable = heredoc.allowIndent 338 ? rawLine.replace(/^\t+/, "") 339 : rawLine; 340 const isDelimiter = comparable === heredoc.delimiter; 341 342 if (lineEnd > lineStart) { 343 pushToken( 344 state, 345 lineStart, 346 lineEnd, 347 "string", 348 isDelimiter 349 ? ["string", "string.heredoc", "string.heredoc.delimiter"] 350 : ["string", "string.heredoc"], 351 ); 352 } 353 354 if (isDelimiter) { 355 state.heredocQueue.shift(); 356 } 357 358 if (state.index < state.content.length) { 359 const newlineStart = state.index; 360 state.index = readLineEnd(state.content, state.index); 361 pushToken(state, newlineStart, state.index, "whitespace"); 362 state.assignmentPending = false; 363 state.canStartComment = true; 364 state.commandWordPending = true; 365 continue; 366 } 367 368 state.assignmentPending = false; 369 state.canStartComment = true; 370 state.commandWordPending = true; 371 return; 372 } 373} 374 375function readNewline(state: BashTokenizerState): void { 376 const start = state.index; 377 state.index = readLineEnd(state.content, state.index); 378 pushToken(state, start, state.index, "whitespace"); 379 state.assignmentPending = false; 380 state.canStartComment = true; 381 state.commandWordPending = true; 382 383 if (state.heredocQueue.length > 0) { 384 consumeHeredocBodies(state); 385 } 386} 387 388function readSingleQuotedString(state: BashTokenizerState): void { 389 const start = state.index; 390 state.index += 1; 391 while ( 392 state.index < state.content.length && 393 state.content[state.index] !== "'" 394 ) { 395 state.index += 1; 396 } 397 if (state.content[state.index] === "'") { 398 state.index += 1; 399 } 400 401 pushToken(state, start, state.index, "string"); 402 if (state.pendingHeredoc) { 403 registerHeredoc( 404 state, 405 state.content.slice(start + 1, Math.max(start + 1, state.index - 1)), 406 ); 407 } else if (!state.assignmentPending && state.commandWordPending) { 408 state.commandWordPending = false; 409 } 410 state.canStartComment = false; 411} 412 413function readCommandSubstitutionEnd( 414 content: string, 415 index: number, 416 open: string, 417 close: string, 418): number { 419 let depth = 1; 420 let cursor = index; 421 422 while (cursor < content.length) { 423 const char = content[cursor]; 424 425 if (char === "'") { 426 cursor += 1; 427 while (cursor < content.length && content[cursor] !== "'") { 428 cursor += 1; 429 } 430 cursor += Number(content[cursor] === "'"); 431 continue; 432 } 433 434 if (char === '"') { 435 cursor += 1; 436 while (cursor < content.length) { 437 if (content[cursor] === "\\") { 438 cursor += 2; 439 continue; 440 } 441 if (content[cursor] === '"') { 442 cursor += 1; 443 break; 444 } 445 cursor += 1; 446 } 447 continue; 448 } 449 450 if (content.startsWith(open, cursor)) { 451 depth += 1; 452 cursor += open.length; 453 continue; 454 } 455 456 if (content.startsWith(close, cursor)) { 457 depth -= 1; 458 cursor += close.length; 459 if (depth === 0) { 460 return cursor; 461 } 462 continue; 463 } 464 465 if (content[cursor] === "\\") { 466 cursor += 2; 467 continue; 468 } 469 470 cursor += 1; 471 } 472 473 return content.length; 474} 475 476function readParameterExpansionEnd(content: string, index: number): number { 477 let depth = 1; 478 let cursor = index; 479 480 while (cursor < content.length) { 481 if (content[cursor] === "\\") { 482 cursor += 2; 483 continue; 484 } 485 if (content[cursor] === "{") { 486 depth += 1; 487 cursor += 1; 488 continue; 489 } 490 if (content[cursor] === "}") { 491 depth -= 1; 492 cursor += 1; 493 if (depth === 0) { 494 return cursor; 495 } 496 continue; 497 } 498 cursor += 1; 499 } 500 501 return content.length; 502} 503 504function readBacktickSubstitution(state: BashTokenizerState): void { 505 const start = state.index; 506 state.index += 1; 507 while (state.index < state.content.length) { 508 if (state.content[state.index] === "\\") { 509 state.index += 2; 510 continue; 511 } 512 if (state.content[state.index] === "`") { 513 state.index += 1; 514 break; 515 } 516 state.index += 1; 517 } 518 519 pushToken(state, start, state.index, "meta", [ 520 "meta", 521 "meta.substitution.command", 522 ]); 523 if (!state.assignmentPending && state.commandWordPending) { 524 state.commandWordPending = false; 525 } 526 state.canStartComment = false; 527} 528 529function readExpansion(state: BashTokenizerState): void { 530 const start = state.index; 531 532 if (state.content.startsWith("$((", state.index)) { 533 state.index = readCommandSubstitutionEnd( 534 state.content, 535 state.index + 3, 536 "((", 537 "))", 538 ); 539 pushToken(state, start, state.index, "meta", [ 540 "meta", 541 "meta.substitution.arithmetic", 542 ]); 543 } else if (state.content.startsWith("$(", state.index)) { 544 state.index = readCommandSubstitutionEnd( 545 state.content, 546 state.index + 2, 547 "(", 548 ")", 549 ); 550 pushToken(state, start, state.index, "meta", [ 551 "meta", 552 "meta.substitution.command", 553 ]); 554 } else if (state.content.startsWith("${", state.index)) { 555 state.index = readParameterExpansionEnd(state.content, state.index + 2); 556 pushToken(state, start, state.index, "variable", [ 557 "variable", 558 "variable.expansion", 559 ]); 560 } else { 561 state.index += 1; 562 const next = state.content[state.index]; 563 if (next && /[A-Za-z0-9_@*#?$!-]/.test(next)) { 564 if (/[A-Za-z_]/.test(next)) { 565 state.index += 1; 566 while (/[A-Za-z0-9_]/.test(state.content[state.index] ?? "")) { 567 state.index += 1; 568 } 569 } else { 570 state.index += 1; 571 } 572 pushToken(state, start, state.index, "variable"); 573 } else { 574 pushToken(state, start, state.index, "text"); 575 } 576 } 577 578 if (!state.assignmentPending && state.commandWordPending) { 579 state.commandWordPending = false; 580 } 581 state.canStartComment = false; 582} 583 584function readEscape(state: BashTokenizerState): void { 585 const start = state.index; 586 state.index += 1; 587 if (state.index < state.content.length) { 588 state.index += 1; 589 } 590 pushToken(state, start, state.index, "escape"); 591 if (!state.assignmentPending && state.commandWordPending) { 592 state.commandWordPending = false; 593 } 594 state.canStartComment = false; 595} 596 597function readDoubleQuotedString(state: BashTokenizerState): void { 598 const start = state.index; 599 let segmentStart = start; 600 state.index += 1; 601 602 while (state.index < state.content.length) { 603 const char = state.content[state.index]; 604 605 if (char === "\\") { 606 if (state.index > segmentStart) { 607 pushToken(state, segmentStart, state.index, "string"); 608 } 609 const escapeStart = state.index; 610 state.index += 1; 611 if (state.index < state.content.length) { 612 state.index += 1; 613 } 614 pushToken(state, escapeStart, state.index, "escape"); 615 segmentStart = state.index; 616 continue; 617 } 618 619 if (char === "$") { 620 if (state.index > segmentStart) { 621 pushToken(state, segmentStart, state.index, "string"); 622 } 623 readExpansion(state); 624 segmentStart = state.index; 625 continue; 626 } 627 628 if (char === "`") { 629 if (state.index > segmentStart) { 630 pushToken(state, segmentStart, state.index, "string"); 631 } 632 readBacktickSubstitution(state); 633 segmentStart = state.index; 634 continue; 635 } 636 637 if (char === '"') { 638 state.index += 1; 639 pushToken(state, segmentStart, state.index, "string"); 640 if (state.pendingHeredoc) { 641 registerHeredoc( 642 state, 643 state.content.slice(start + 1, Math.max(start + 1, state.index - 1)), 644 ); 645 } else if (!state.assignmentPending && state.commandWordPending) { 646 state.commandWordPending = false; 647 } 648 state.canStartComment = false; 649 return; 650 } 651 652 state.index += 1; 653 } 654 655 pushToken(state, segmentStart, state.index, "string"); 656 if (state.pendingHeredoc) { 657 registerHeredoc(state, state.content.slice(start + 1, state.index)); 658 } else if (!state.assignmentPending && state.commandWordPending) { 659 state.commandWordPending = false; 660 } 661 state.canStartComment = false; 662} 663 664function readOperator(state: BashTokenizerState, operator: string): void { 665 const start = state.index; 666 state.index += operator.length; 667 pushToken(state, start, state.index, "operator"); 668 state.assignmentPending = false; 669 state.canStartComment = true; 670 671 if (operator === "<<" || operator === "<<-") { 672 state.pendingHeredoc = { allowIndent: operator === "<<-" }; 673 } 674 675 if (COMMAND_BOUNDARY_OPERATORS.has(operator)) { 676 state.commandWordPending = true; 677 state.expectForIn = false; 678 state.expectCaseIn = false; 679 } 680} 681 682function readPlainSegment(state: BashTokenizerState): void { 683 const start = state.index; 684 const end = readPlainSegmentEnd(state.content, state.index); 685 const text = state.content.slice(start, end); 686 state.index = end; 687 688 if (state.pendingHeredoc) { 689 pushToken(state, start, end, "string", [ 690 "string", 691 "string.heredoc.delimiter", 692 ]); 693 registerHeredoc(state, text); 694 state.canStartComment = false; 695 return; 696 } 697 698 if (state.expectFunctionName) { 699 pushToken(state, start, end, "function"); 700 state.expectFunctionName = false; 701 state.commandWordPending = false; 702 state.canStartComment = false; 703 return; 704 } 705 706 if (state.expectForName) { 707 pushToken(state, start, end, "variable"); 708 state.expectForName = false; 709 state.expectForIn = true; 710 state.commandWordPending = false; 711 state.canStartComment = false; 712 return; 713 } 714 715 if (state.expectCaseSubject) { 716 pushToken(state, start, end, "text"); 717 state.expectCaseSubject = false; 718 state.expectCaseIn = true; 719 state.commandWordPending = false; 720 state.canStartComment = false; 721 return; 722 } 723 724 if ((state.expectForIn || state.expectCaseIn) && text === "in") { 725 pushKeyword(state, start, end); 726 state.canStartComment = false; 727 return; 728 } 729 730 if (state.assignmentPending) { 731 if (isNumber(text)) { 732 pushToken(state, start, end, "number"); 733 } else { 734 pushToken(state, start, end, "text"); 735 } 736 state.canStartComment = false; 737 return; 738 } 739 740 if (BASH_KEYWORDS.has(text) && (state.commandWordPending || text === "in")) { 741 pushKeyword(state, start, end); 742 state.canStartComment = false; 743 return; 744 } 745 746 if (state.commandWordPending && isAssignmentPrefix(text)) { 747 pushToken(state, start, end - 1, "variable"); 748 pushToken(state, end - 1, end, "operator"); 749 state.assignmentPending = true; 750 state.canStartComment = false; 751 return; 752 } 753 754 if (state.commandWordPending && state.content.startsWith("()", state.index)) { 755 pushCommandLike(state, start, end, "function"); 756 state.canStartComment = false; 757 return; 758 } 759 760 if (state.commandWordPending && BASH_BUILTINS.has(text)) { 761 pushCommandLike(state, start, end, "builtin"); 762 state.canStartComment = false; 763 return; 764 } 765 766 if (state.commandWordPending) { 767 pushCommandLike(state, start, end, "command"); 768 state.canStartComment = false; 769 return; 770 } 771 772 if (isNumber(text)) { 773 pushToken(state, start, end, "number"); 774 } else { 775 pushToken(state, start, end, "text"); 776 } 777 state.canStartComment = false; 778} 779 780function tokenizeBash(content: string): ClewOutput { 781 const state: BashTokenizerState = { 782 assignmentPending: false, 783 canStartComment: true, 784 commandWordPending: true, 785 content, 786 expectCaseIn: false, 787 expectCaseSubject: false, 788 expectForIn: false, 789 expectForName: false, 790 expectFunctionName: false, 791 heredocQueue: [], 792 index: 0, 793 pendingHeredoc: null, 794 tokens: [], 795 }; 796 797 while (state.index < content.length) { 798 const char = state.content[state.index]; 799 if (isSpace(char)) { 800 readSpaces(state); 801 continue; 802 } 803 if (isNewlineStart(state.content, state.index)) { 804 readNewline(state); 805 continue; 806 } 807 if (char === "#" && state.canStartComment) { 808 readComment(state); 809 continue; 810 } 811 812 const operator = matchOperator(state.content, state.index); 813 if (operator) { 814 readOperator(state, operator); 815 continue; 816 } 817 818 if (char === "'") { 819 readSingleQuotedString(state); 820 continue; 821 } 822 if (char === '"') { 823 readDoubleQuotedString(state); 824 continue; 825 } 826 if (char === "$") { 827 readExpansion(state); 828 continue; 829 } 830 if (char === "`") { 831 readBacktickSubstitution(state); 832 continue; 833 } 834 if (char === "\\") { 835 readEscape(state); 836 continue; 837 } 838 839 readPlainSegment(state); 840 } 841 842 return { tokens: state.tokens }; 843} 844 845export const bashLanguageSupport: ClewLanguageSupport = { 846 ids: ["bash"], 847 stableBoundary: stableLineBoundary, 848 tokenize: tokenizeBash, 849};