xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / languages / bash.ts

18.7 KB
1import type { ClewOutput, ClewToken } from "../model.js";
2import type { ClewLanguageSupport } from "../registry.js";
3import { pushTokenRange, stableLineBoundary } from "../shared.js";
4
5const BASH_KEYWORDS = new Set([
6  "if",
7  "then",
8  "elif",
9  "else",
10  "fi",
11  "for",
12  "while",
13  "until",
14  "do",
15  "done",
16  "case",
17  "esac",
18  "select",
19  "function",
20  "in",
21  "time",
22  "coproc",
23]);
24
25const COMMAND_FOLLOWING_KEYWORDS = new Set([
26  "if",
27  "then",
28  "elif",
29  "else",
30  "while",
31  "until",
32  "do",
33  "time",
34  "coproc",
35]);
36
37const BASH_BUILTINS = new Set([
38  ".",
39  ":",
40  "alias",
41  "bg",
42  "bind",
43  "break",
44  "builtin",
45  "caller",
46  "cd",
47  "command",
48  "compgen",
49  "complete",
50  "compopt",
51  "continue",
52  "declare",
53  "dirs",
54  "disown",
55  "echo",
56  "enable",
57  "eval",
58  "exec",
59  "exit",
60  "export",
61  "false",
62  "fc",
63  "fg",
64  "getopts",
65  "hash",
66  "help",
67  "history",
68  "jobs",
69  "kill",
70  "let",
71  "local",
72  "logout",
73  "mapfile",
74  "popd",
75  "printf",
76  "pushd",
77  "pwd",
78  "read",
79  "readarray",
80  "readonly",
81  "return",
82  "set",
83  "shift",
84  "shopt",
85  "source",
86  "suspend",
87  "test",
88  "times",
89  "trap",
90  "true",
91  "type",
92  "typeset",
93  "ulimit",
94  "umask",
95  "unalias",
96  "unset",
97  "wait",
98]);
99
100const COMMAND_BOUNDARY_OPERATORS = new Set([
101  "&",
102  "&&",
103  "(",
104  ";",
105  ";;",
106  ";&",
107  ";;&",
108  "|",
109  "|&",
110  "||",
111  "{",
112]);
113
114const OPERATORS = [
115  "<<-",
116  "<<<",
117  "<<",
118  ">>",
119  "||",
120  "&&",
121  "|&",
122  "&>>",
123  "&>",
124  ">&",
125  "<&",
126  "<>",
127  ">|",
128  ";;&",
129  ";&",
130  ";;",
131  "[[",
132  "]]",
133  "|",
134  "&",
135  ";",
136  "(",
137  ")",
138  "{",
139  "}",
140  "<",
141  ">",
142] as const;
143
144interface PendingHeredoc {
145  allowIndent: boolean;
146}
147
148interface HeredocSpec {
149  allowIndent: boolean;
150  delimiter: string;
151}
152
153interface BashTokenizerState {
154  assignmentPending: boolean;
155  canStartComment: boolean;
156  commandWordPending: boolean;
157  content: string;
158  expectCaseIn: boolean;
159  expectCaseSubject: boolean;
160  expectForIn: boolean;
161  expectForName: boolean;
162  expectFunctionName: boolean;
163  heredocQueue: HeredocSpec[];
164  index: number;
165  pendingHeredoc: PendingHeredoc | null;
166  tokens: ClewToken[];
167}
168
169function isSpace(char: string | undefined): boolean {
170  return char === " " || char === "\t";
171}
172
173function isNewlineStart(content: string, index: number): boolean {
174  return content[index] === "\n" || content[index] === "\r";
175}
176
177function isWordBoundaryStart(content: string, index: number): boolean {
178  const char = content[index];
179  if (char === undefined) {
180    return true;
181  }
182
183  return (
184    char === "$" ||
185    char === '"' ||
186    char === "'" ||
187    char === "`" ||
188    char === "\\" ||
189    isSpace(char) ||
190    isNewlineStart(content, index) ||
191    OPERATORS.some((operator) => content.startsWith(operator, index))
192  );
193}
194
195function pushToken(
196  state: BashTokenizerState,
197  start: number,
198  end: number,
199  type: string,
200  scopes?: readonly string[],
201): void {
202  pushTokenRange(state.tokens, state.content, start, end, type, scopes);
203}
204
205function readWhile(
206  state: BashTokenizerState,
207  predicate: (char: string | undefined, index: number) => boolean,
208): number {
209  const start = state.index;
210  while (predicate(state.content[state.index], state.index)) {
211    state.index += 1;
212  }
213  return start;
214}
215
216function readLineEnd(content: string, index: number): number {
217  if (content[index] === "\r" && content[index + 1] === "\n") {
218    return index + 2;
219  }
220  return index + 1;
221}
222
223function matchOperator(content: string, index: number): string | undefined {
224  return OPERATORS.find((operator) => content.startsWith(operator, index));
225}
226
227function readPlainSegmentEnd(content: string, index: number): number {
228  let end = index;
229  while (!isWordBoundaryStart(content, end)) {
230    end += 1;
231  }
232  return end;
233}
234
235function isAssignmentPrefix(text: string): boolean {
236  return /^[A-Za-z_][A-Za-z0-9_]*=$/.test(text);
237}
238
239function isNumber(text: string): boolean {
240  return /^(?:\d+|0x[0-9a-fA-F]+)$/.test(text);
241}
242
243function registerHeredoc(
244  state: BashTokenizerState,
245  rawDelimiter: string,
246): void {
247  if (!state.pendingHeredoc) {
248    return;
249  }
250
251  state.heredocQueue.push({
252    allowIndent: state.pendingHeredoc.allowIndent,
253    delimiter: rawDelimiter,
254  });
255  state.pendingHeredoc = null;
256}
257
258function classifyKeywordState(state: BashTokenizerState, text: string): void {
259  state.expectFunctionName = text === "function";
260  state.expectForName = text === "for" || text === "select";
261  state.expectCaseSubject = text === "case";
262
263  if (text === "in") {
264    state.expectForIn = false;
265    state.expectCaseIn = false;
266  } else if (!state.expectForName) {
267    state.expectForIn = state.expectForIn && text !== "for";
268  }
269
270  if (text !== "case") {
271    state.expectCaseIn = state.expectCaseIn && text !== "in";
272  }
273
274  state.assignmentPending = false;
275  state.commandWordPending = COMMAND_FOLLOWING_KEYWORDS.has(text);
276}
277
278function pushKeyword(
279  state: BashTokenizerState,
280  start: number,
281  end: number,
282): void {
283  pushToken(state, start, end, "keyword");
284  classifyKeywordState(state, state.content.slice(start, end));
285}
286
287function pushCommandLike(
288  state: BashTokenizerState,
289  start: number,
290  end: number,
291  type: "builtin" | "command" | "function" | "number" | "text" | "variable",
292  scopes?: readonly string[],
293): void {
294  pushToken(state, start, end, type, scopes);
295  state.assignmentPending = false;
296  state.commandWordPending = false;
297}
298
299function readComment(state: BashTokenizerState): void {
300  const start = state.index;
301  while (
302    state.index < state.content.length &&
303    !isNewlineStart(state.content, state.index)
304  ) {
305    state.index += 1;
306  }
307  pushToken(state, start, state.index, "comment");
308  state.assignmentPending = false;
309  state.canStartComment = false;
310  state.commandWordPending = false;
311}
312
313function readSpaces(state: BashTokenizerState): void {
314  const start = readWhile(state, (char) => isSpace(char));
315  pushToken(state, start, state.index, "whitespace");
316  state.assignmentPending = false;
317  state.canStartComment = true;
318}
319
320function consumeHeredocBodies(state: BashTokenizerState): void {
321  while (state.heredocQueue.length > 0 && state.index < state.content.length) {
322    const heredoc = state.heredocQueue[0];
323    if (!heredoc) {
324      return;
325    }
326
327    const lineStart = state.index;
328    while (
329      state.index < state.content.length &&
330      !isNewlineStart(state.content, state.index)
331    ) {
332      state.index += 1;
333    }
334
335    const lineEnd = state.index;
336    const rawLine = state.content.slice(lineStart, lineEnd);
337    const comparable = heredoc.allowIndent
338      ? rawLine.replace(/^\t+/, "")
339      : rawLine;
340    const isDelimiter = comparable === heredoc.delimiter;
341
342    if (lineEnd > lineStart) {
343      pushToken(
344        state,
345        lineStart,
346        lineEnd,
347        "string",
348        isDelimiter
349          ? ["string", "string.heredoc", "string.heredoc.delimiter"]
350          : ["string", "string.heredoc"],
351      );
352    }
353
354    if (isDelimiter) {
355      state.heredocQueue.shift();
356    }
357
358    if (state.index < state.content.length) {
359      const newlineStart = state.index;
360      state.index = readLineEnd(state.content, state.index);
361      pushToken(state, newlineStart, state.index, "whitespace");
362      state.assignmentPending = false;
363      state.canStartComment = true;
364      state.commandWordPending = true;
365      continue;
366    }
367
368    state.assignmentPending = false;
369    state.canStartComment = true;
370    state.commandWordPending = true;
371    return;
372  }
373}
374
375function readNewline(state: BashTokenizerState): void {
376  const start = state.index;
377  state.index = readLineEnd(state.content, state.index);
378  pushToken(state, start, state.index, "whitespace");
379  state.assignmentPending = false;
380  state.canStartComment = true;
381  state.commandWordPending = true;
382
383  if (state.heredocQueue.length > 0) {
384    consumeHeredocBodies(state);
385  }
386}
387
388function readSingleQuotedString(state: BashTokenizerState): void {
389  const start = state.index;
390  state.index += 1;
391  while (
392    state.index < state.content.length &&
393    state.content[state.index] !== "'"
394  ) {
395    state.index += 1;
396  }
397  if (state.content[state.index] === "'") {
398    state.index += 1;
399  }
400
401  pushToken(state, start, state.index, "string");
402  if (state.pendingHeredoc) {
403    registerHeredoc(
404      state,
405      state.content.slice(start + 1, Math.max(start + 1, state.index - 1)),
406    );
407  } else if (!state.assignmentPending && state.commandWordPending) {
408    state.commandWordPending = false;
409  }
410  state.canStartComment = false;
411}
412
413function readCommandSubstitutionEnd(
414  content: string,
415  index: number,
416  open: string,
417  close: string,
418): number {
419  let depth = 1;
420  let cursor = index;
421
422  while (cursor < content.length) {
423    const char = content[cursor];
424
425    if (char === "'") {
426      cursor += 1;
427      while (cursor < content.length && content[cursor] !== "'") {
428        cursor += 1;
429      }
430      cursor += Number(content[cursor] === "'");
431      continue;
432    }
433
434    if (char === '"') {
435      cursor += 1;
436      while (cursor < content.length) {
437        if (content[cursor] === "\\") {
438          cursor += 2;
439          continue;
440        }
441        if (content[cursor] === '"') {
442          cursor += 1;
443          break;
444        }
445        cursor += 1;
446      }
447      continue;
448    }
449
450    if (content.startsWith(open, cursor)) {
451      depth += 1;
452      cursor += open.length;
453      continue;
454    }
455
456    if (content.startsWith(close, cursor)) {
457      depth -= 1;
458      cursor += close.length;
459      if (depth === 0) {
460        return cursor;
461      }
462      continue;
463    }
464
465    if (content[cursor] === "\\") {
466      cursor += 2;
467      continue;
468    }
469
470    cursor += 1;
471  }
472
473  return content.length;
474}
475
476function readParameterExpansionEnd(content: string, index: number): number {
477  let depth = 1;
478  let cursor = index;
479
480  while (cursor < content.length) {
481    if (content[cursor] === "\\") {
482      cursor += 2;
483      continue;
484    }
485    if (content[cursor] === "{") {
486      depth += 1;
487      cursor += 1;
488      continue;
489    }
490    if (content[cursor] === "}") {
491      depth -= 1;
492      cursor += 1;
493      if (depth === 0) {
494        return cursor;
495      }
496      continue;
497    }
498    cursor += 1;
499  }
500
501  return content.length;
502}
503
504function readBacktickSubstitution(state: BashTokenizerState): void {
505  const start = state.index;
506  state.index += 1;
507  while (state.index < state.content.length) {
508    if (state.content[state.index] === "\\") {
509      state.index += 2;
510      continue;
511    }
512    if (state.content[state.index] === "`") {
513      state.index += 1;
514      break;
515    }
516    state.index += 1;
517  }
518
519  pushToken(state, start, state.index, "meta", [
520    "meta",
521    "meta.substitution.command",
522  ]);
523  if (!state.assignmentPending && state.commandWordPending) {
524    state.commandWordPending = false;
525  }
526  state.canStartComment = false;
527}
528
529function readExpansion(state: BashTokenizerState): void {
530  const start = state.index;
531
532  if (state.content.startsWith("$((", state.index)) {
533    state.index = readCommandSubstitutionEnd(
534      state.content,
535      state.index + 3,
536      "((",
537      "))",
538    );
539    pushToken(state, start, state.index, "meta", [
540      "meta",
541      "meta.substitution.arithmetic",
542    ]);
543  } else if (state.content.startsWith("$(", state.index)) {
544    state.index = readCommandSubstitutionEnd(
545      state.content,
546      state.index + 2,
547      "(",
548      ")",
549    );
550    pushToken(state, start, state.index, "meta", [
551      "meta",
552      "meta.substitution.command",
553    ]);
554  } else if (state.content.startsWith("${", state.index)) {
555    state.index = readParameterExpansionEnd(state.content, state.index + 2);
556    pushToken(state, start, state.index, "variable", [
557      "variable",
558      "variable.expansion",
559    ]);
560  } else {
561    state.index += 1;
562    const next = state.content[state.index];
563    if (next && /[A-Za-z0-9_@*#?$!-]/.test(next)) {
564      if (/[A-Za-z_]/.test(next)) {
565        state.index += 1;
566        while (/[A-Za-z0-9_]/.test(state.content[state.index] ?? "")) {
567          state.index += 1;
568        }
569      } else {
570        state.index += 1;
571      }
572      pushToken(state, start, state.index, "variable");
573    } else {
574      pushToken(state, start, state.index, "text");
575    }
576  }
577
578  if (!state.assignmentPending && state.commandWordPending) {
579    state.commandWordPending = false;
580  }
581  state.canStartComment = false;
582}
583
584function readEscape(state: BashTokenizerState): void {
585  const start = state.index;
586  state.index += 1;
587  if (state.index < state.content.length) {
588    state.index += 1;
589  }
590  pushToken(state, start, state.index, "escape");
591  if (!state.assignmentPending && state.commandWordPending) {
592    state.commandWordPending = false;
593  }
594  state.canStartComment = false;
595}
596
597function readDoubleQuotedString(state: BashTokenizerState): void {
598  const start = state.index;
599  let segmentStart = start;
600  state.index += 1;
601
602  while (state.index < state.content.length) {
603    const char = state.content[state.index];
604
605    if (char === "\\") {
606      if (state.index > segmentStart) {
607        pushToken(state, segmentStart, state.index, "string");
608      }
609      const escapeStart = state.index;
610      state.index += 1;
611      if (state.index < state.content.length) {
612        state.index += 1;
613      }
614      pushToken(state, escapeStart, state.index, "escape");
615      segmentStart = state.index;
616      continue;
617    }
618
619    if (char === "$") {
620      if (state.index > segmentStart) {
621        pushToken(state, segmentStart, state.index, "string");
622      }
623      readExpansion(state);
624      segmentStart = state.index;
625      continue;
626    }
627
628    if (char === "`") {
629      if (state.index > segmentStart) {
630        pushToken(state, segmentStart, state.index, "string");
631      }
632      readBacktickSubstitution(state);
633      segmentStart = state.index;
634      continue;
635    }
636
637    if (char === '"') {
638      state.index += 1;
639      pushToken(state, segmentStart, state.index, "string");
640      if (state.pendingHeredoc) {
641        registerHeredoc(
642          state,
643          state.content.slice(start + 1, Math.max(start + 1, state.index - 1)),
644        );
645      } else if (!state.assignmentPending && state.commandWordPending) {
646        state.commandWordPending = false;
647      }
648      state.canStartComment = false;
649      return;
650    }
651
652    state.index += 1;
653  }
654
655  pushToken(state, segmentStart, state.index, "string");
656  if (state.pendingHeredoc) {
657    registerHeredoc(state, state.content.slice(start + 1, state.index));
658  } else if (!state.assignmentPending && state.commandWordPending) {
659    state.commandWordPending = false;
660  }
661  state.canStartComment = false;
662}
663
664function readOperator(state: BashTokenizerState, operator: string): void {
665  const start = state.index;
666  state.index += operator.length;
667  pushToken(state, start, state.index, "operator");
668  state.assignmentPending = false;
669  state.canStartComment = true;
670
671  if (operator === "<<" || operator === "<<-") {
672    state.pendingHeredoc = { allowIndent: operator === "<<-" };
673  }
674
675  if (COMMAND_BOUNDARY_OPERATORS.has(operator)) {
676    state.commandWordPending = true;
677    state.expectForIn = false;
678    state.expectCaseIn = false;
679  }
680}
681
682function readPlainSegment(state: BashTokenizerState): void {
683  const start = state.index;
684  const end = readPlainSegmentEnd(state.content, state.index);
685  const text = state.content.slice(start, end);
686  state.index = end;
687
688  if (state.pendingHeredoc) {
689    pushToken(state, start, end, "string", [
690      "string",
691      "string.heredoc.delimiter",
692    ]);
693    registerHeredoc(state, text);
694    state.canStartComment = false;
695    return;
696  }
697
698  if (state.expectFunctionName) {
699    pushToken(state, start, end, "function");
700    state.expectFunctionName = false;
701    state.commandWordPending = false;
702    state.canStartComment = false;
703    return;
704  }
705
706  if (state.expectForName) {
707    pushToken(state, start, end, "variable");
708    state.expectForName = false;
709    state.expectForIn = true;
710    state.commandWordPending = false;
711    state.canStartComment = false;
712    return;
713  }
714
715  if (state.expectCaseSubject) {
716    pushToken(state, start, end, "text");
717    state.expectCaseSubject = false;
718    state.expectCaseIn = true;
719    state.commandWordPending = false;
720    state.canStartComment = false;
721    return;
722  }
723
724  if ((state.expectForIn || state.expectCaseIn) && text === "in") {
725    pushKeyword(state, start, end);
726    state.canStartComment = false;
727    return;
728  }
729
730  if (state.assignmentPending) {
731    if (isNumber(text)) {
732      pushToken(state, start, end, "number");
733    } else {
734      pushToken(state, start, end, "text");
735    }
736    state.canStartComment = false;
737    return;
738  }
739
740  if (BASH_KEYWORDS.has(text) && (state.commandWordPending || text === "in")) {
741    pushKeyword(state, start, end);
742    state.canStartComment = false;
743    return;
744  }
745
746  if (state.commandWordPending && isAssignmentPrefix(text)) {
747    pushToken(state, start, end - 1, "variable");
748    pushToken(state, end - 1, end, "operator");
749    state.assignmentPending = true;
750    state.canStartComment = false;
751    return;
752  }
753
754  if (state.commandWordPending && state.content.startsWith("()", state.index)) {
755    pushCommandLike(state, start, end, "function");
756    state.canStartComment = false;
757    return;
758  }
759
760  if (state.commandWordPending && BASH_BUILTINS.has(text)) {
761    pushCommandLike(state, start, end, "builtin");
762    state.canStartComment = false;
763    return;
764  }
765
766  if (state.commandWordPending) {
767    pushCommandLike(state, start, end, "command");
768    state.canStartComment = false;
769    return;
770  }
771
772  if (isNumber(text)) {
773    pushToken(state, start, end, "number");
774  } else {
775    pushToken(state, start, end, "text");
776  }
777  state.canStartComment = false;
778}
779
780function tokenizeBash(content: string): ClewOutput {
781  const state: BashTokenizerState = {
782    assignmentPending: false,
783    canStartComment: true,
784    commandWordPending: true,
785    content,
786    expectCaseIn: false,
787    expectCaseSubject: false,
788    expectForIn: false,
789    expectForName: false,
790    expectFunctionName: false,
791    heredocQueue: [],
792    index: 0,
793    pendingHeredoc: null,
794    tokens: [],
795  };
796
797  while (state.index < content.length) {
798    const char = state.content[state.index];
799    if (isSpace(char)) {
800      readSpaces(state);
801      continue;
802    }
803    if (isNewlineStart(state.content, state.index)) {
804      readNewline(state);
805      continue;
806    }
807    if (char === "#" && state.canStartComment) {
808      readComment(state);
809      continue;
810    }
811
812    const operator = matchOperator(state.content, state.index);
813    if (operator) {
814      readOperator(state, operator);
815      continue;
816    }
817
818    if (char === "'") {
819      readSingleQuotedString(state);
820      continue;
821    }
822    if (char === '"') {
823      readDoubleQuotedString(state);
824      continue;
825    }
826    if (char === "$") {
827      readExpansion(state);
828      continue;
829    }
830    if (char === "`") {
831      readBacktickSubstitution(state);
832      continue;
833    }
834    if (char === "\\") {
835      readEscape(state);
836      continue;
837    }
838
839    readPlainSegment(state);
840  }
841
842  return { tokens: state.tokens };
843}
844
845export const bashLanguageSupport: ClewLanguageSupport = {
846  ids: ["bash"],
847  stableBoundary: stableLineBoundary,
848  tokenize: tokenizeBash,
849};