xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / languages / json.ts

5.8 KB
1import type { ClewOutput, ClewToken } from "../model.js";
2import type { ClewLanguageSupport } from "../registry.js";
3import { pushTokenRange, stableLineBoundary } from "../shared.js";
4
5type JsonContext =
6  | {
7      kind: "array";
8      state: "valueOrEnd" | "commaOrEnd";
9    }
10  | {
11      kind: "object";
12      state: "keyOrEnd" | "colon" | "valueOrEnd" | "commaOrEnd";
13    };
14
15interface JsonTokenizerState {
16  content: string;
17  index: number;
18  rootValueComplete: boolean;
19  stack: JsonContext[];
20  tokens: ClewToken[];
21}
22
23function currentContext(state: JsonTokenizerState): JsonContext | undefined {
24  return state.stack.at(-1);
25}
26
27function pushToken(
28  state: JsonTokenizerState,
29  start: number,
30  end: number,
31  type: string,
32  scopes?: readonly string[],
33): void {
34  pushTokenRange(state.tokens, state.content, start, end, type, scopes);
35}
36
37function completeValue(state: JsonTokenizerState): void {
38  const context = currentContext(state);
39  if (!context) {
40    state.rootValueComplete = true;
41    return;
42  }
43
44  context.state = "commaOrEnd";
45}
46
47function readStringEnd(content: string, start: number): number {
48  let index = start + 1;
49
50  while (index < content.length) {
51    const char = content[index];
52
53    if (char === "\\") {
54      index += 2;
55      continue;
56    }
57
58    if (char === '"') {
59      return index + 1;
60    }
61
62    index += 1;
63  }
64
65  return content.length;
66}
67
68function isDigit(char: string | undefined): boolean {
69  return char !== undefined && char >= "0" && char <= "9";
70}
71
72function readNumberEnd(content: string, start: number): number {
73  let index = start;
74
75  if (content[index] === "-") {
76    index += 1;
77  }
78
79  if (content[index] === "0") {
80    index += 1;
81  } else {
82    while (isDigit(content[index])) {
83      index += 1;
84    }
85  }
86
87  if (content[index] === ".") {
88    index += 1;
89    while (isDigit(content[index])) {
90      index += 1;
91    }
92  }
93
94  const exponent = content[index];
95  if (exponent === "e" || exponent === "E") {
96    index += 1;
97    const sign = content[index];
98    if (sign === "+" || sign === "-") {
99      index += 1;
100    }
101    while (isDigit(content[index])) {
102      index += 1;
103    }
104  }
105
106  return index;
107}
108
109function consumeKeyword(
110  state: JsonTokenizerState,
111  keyword: string,
112  scopes?: readonly string[],
113): boolean {
114  if (!state.content.startsWith(keyword, state.index)) {
115    return false;
116  }
117
118  const end = state.index + keyword.length;
119  pushToken(state, state.index, end, "keyword", scopes ?? ["keyword"]);
120  state.index = end;
121  completeValue(state);
122  return true;
123}
124
125function tokenizeJson(content: string): ClewOutput {
126  const state: JsonTokenizerState = {
127    content,
128    index: 0,
129    rootValueComplete: false,
130    stack: [],
131    tokens: [],
132  };
133
134  while (state.index < content.length) {
135    const char = content[state.index];
136
137    if (char === undefined) {
138      break;
139    }
140
141    if (/\s/.test(char)) {
142      const start = state.index;
143      while (/\s/.test(content[state.index] ?? "")) {
144        state.index += 1;
145      }
146      pushToken(state, start, state.index, "whitespace");
147      continue;
148    }
149
150    const context = currentContext(state);
151
152    if (char === "{") {
153      pushToken(state, state.index, state.index + 1, "punctuation");
154      state.index += 1;
155      state.stack.push({ kind: "object", state: "keyOrEnd" });
156      continue;
157    }
158
159    if (char === "[") {
160      pushToken(state, state.index, state.index + 1, "punctuation");
161      state.index += 1;
162      state.stack.push({ kind: "array", state: "valueOrEnd" });
163      continue;
164    }
165
166    if (char === "}") {
167      pushToken(state, state.index, state.index + 1, "punctuation");
168      state.index += 1;
169      if (context?.kind === "object") {
170        state.stack.pop();
171        completeValue(state);
172      }
173      continue;
174    }
175
176    if (char === "]") {
177      pushToken(state, state.index, state.index + 1, "punctuation");
178      state.index += 1;
179      if (context?.kind === "array") {
180        state.stack.pop();
181        completeValue(state);
182      }
183      continue;
184    }
185
186    if (char === ":") {
187      pushToken(state, state.index, state.index + 1, "punctuation");
188      state.index += 1;
189      if (context?.kind === "object" && context.state === "colon") {
190        context.state = "valueOrEnd";
191      }
192      continue;
193    }
194
195    if (char === ",") {
196      pushToken(state, state.index, state.index + 1, "punctuation");
197      state.index += 1;
198      if (context?.kind === "object" && context.state === "commaOrEnd") {
199        context.state = "keyOrEnd";
200      } else if (context?.kind === "array" && context.state === "commaOrEnd") {
201        context.state = "valueOrEnd";
202      }
203      continue;
204    }
205
206    if (char === '"') {
207      const start = state.index;
208      const end = readStringEnd(content, start);
209      const isProperty =
210        context?.kind === "object" && context.state === "keyOrEnd";
211      pushToken(
212        state,
213        start,
214        end,
215        "string",
216        isProperty ? ["string", "property"] : ["string"],
217      );
218      state.index = end;
219
220      if (isProperty && context.kind === "object") {
221        context.state = "colon";
222      } else {
223        completeValue(state);
224      }
225      continue;
226    }
227
228    if (char === "-" || isDigit(char)) {
229      const end = readNumberEnd(content, state.index);
230      if (end > state.index) {
231        pushToken(state, state.index, end, "number");
232        state.index = end;
233        completeValue(state);
234        continue;
235      }
236    }
237
238    if (
239      consumeKeyword(state, "true", ["keyword", "constant.boolean"]) ||
240      consumeKeyword(state, "false", ["keyword", "constant.boolean"]) ||
241      consumeKeyword(state, "null", ["keyword", "constant.null"])
242    ) {
243      continue;
244    }
245
246    pushToken(state, state.index, state.index + 1, "text");
247    state.index += 1;
248  }
249
250  return { tokens: state.tokens };
251}
252
253export const jsonLanguageSupport: ClewLanguageSupport = {
254  ids: ["json"],
255  stableBoundary: stableLineBoundary,
256  tokenize: tokenizeJson,
257};