70160
Ce
cel-tui / files
cel-tui / packages / clew / src / languages / json.ts
5.8 KB
1import type { ClewOutput, ClewToken } from "../model.js"; 2import type { ClewLanguageSupport } from "../registry.js"; 3import { pushTokenRange, stableLineBoundary } from "../shared.js"; 4 5type JsonContext = 6 | { 7 kind: "array"; 8 state: "valueOrEnd" | "commaOrEnd"; 9 } 10 | { 11 kind: "object"; 12 state: "keyOrEnd" | "colon" | "valueOrEnd" | "commaOrEnd"; 13 }; 14 15interface JsonTokenizerState { 16 content: string; 17 index: number; 18 rootValueComplete: boolean; 19 stack: JsonContext[]; 20 tokens: ClewToken[]; 21} 22 23function currentContext(state: JsonTokenizerState): JsonContext | undefined { 24 return state.stack.at(-1); 25} 26 27function pushToken( 28 state: JsonTokenizerState, 29 start: number, 30 end: number, 31 type: string, 32 scopes?: readonly string[], 33): void { 34 pushTokenRange(state.tokens, state.content, start, end, type, scopes); 35} 36 37function completeValue(state: JsonTokenizerState): void { 38 const context = currentContext(state); 39 if (!context) { 40 state.rootValueComplete = true; 41 return; 42 } 43 44 context.state = "commaOrEnd"; 45} 46 47function readStringEnd(content: string, start: number): number { 48 let index = start + 1; 49 50 while (index < content.length) { 51 const char = content[index]; 52 53 if (char === "\\") { 54 index += 2; 55 continue; 56 } 57 58 if (char === '"') { 59 return index + 1; 60 } 61 62 index += 1; 63 } 64 65 return content.length; 66} 67 68function isDigit(char: string | undefined): boolean { 69 return char !== undefined && char >= "0" && char <= "9"; 70} 71 72function readNumberEnd(content: string, start: number): number { 73 let index = start; 74 75 if (content[index] === "-") { 76 index += 1; 77 } 78 79 if (content[index] === "0") { 80 index += 1; 81 } else { 82 while (isDigit(content[index])) { 83 index += 1; 84 } 85 } 86 87 if (content[index] === ".") { 88 index += 1; 89 while (isDigit(content[index])) { 90 index += 1; 91 } 92 } 93 94 const exponent = content[index]; 95 if (exponent === "e" || exponent === "E") { 96 index += 1; 97 const sign = content[index]; 98 if (sign === "+" || sign === "-") { 99 index += 1; 100 } 101 while (isDigit(content[index])) { 102 index += 1; 103 } 104 } 105 106 return index; 107} 108 109function consumeKeyword( 110 state: JsonTokenizerState, 111 keyword: string, 112 scopes?: readonly string[], 113): boolean { 114 if (!state.content.startsWith(keyword, state.index)) { 115 return false; 116 } 117 118 const end = state.index + keyword.length; 119 pushToken(state, state.index, end, "keyword", scopes ?? ["keyword"]); 120 state.index = end; 121 completeValue(state); 122 return true; 123} 124 125function tokenizeJson(content: string): ClewOutput { 126 const state: JsonTokenizerState = { 127 content, 128 index: 0, 129 rootValueComplete: false, 130 stack: [], 131 tokens: [], 132 }; 133 134 while (state.index < content.length) { 135 const char = content[state.index]; 136 137 if (char === undefined) { 138 break; 139 } 140 141 if (/\s/.test(char)) { 142 const start = state.index; 143 while (/\s/.test(content[state.index] ?? "")) { 144 state.index += 1; 145 } 146 pushToken(state, start, state.index, "whitespace"); 147 continue; 148 } 149 150 const context = currentContext(state); 151 152 if (char === "{") { 153 pushToken(state, state.index, state.index + 1, "punctuation"); 154 state.index += 1; 155 state.stack.push({ kind: "object", state: "keyOrEnd" }); 156 continue; 157 } 158 159 if (char === "[") { 160 pushToken(state, state.index, state.index + 1, "punctuation"); 161 state.index += 1; 162 state.stack.push({ kind: "array", state: "valueOrEnd" }); 163 continue; 164 } 165 166 if (char === "}") { 167 pushToken(state, state.index, state.index + 1, "punctuation"); 168 state.index += 1; 169 if (context?.kind === "object") { 170 state.stack.pop(); 171 completeValue(state); 172 } 173 continue; 174 } 175 176 if (char === "]") { 177 pushToken(state, state.index, state.index + 1, "punctuation"); 178 state.index += 1; 179 if (context?.kind === "array") { 180 state.stack.pop(); 181 completeValue(state); 182 } 183 continue; 184 } 185 186 if (char === ":") { 187 pushToken(state, state.index, state.index + 1, "punctuation"); 188 state.index += 1; 189 if (context?.kind === "object" && context.state === "colon") { 190 context.state = "valueOrEnd"; 191 } 192 continue; 193 } 194 195 if (char === ",") { 196 pushToken(state, state.index, state.index + 1, "punctuation"); 197 state.index += 1; 198 if (context?.kind === "object" && context.state === "commaOrEnd") { 199 context.state = "keyOrEnd"; 200 } else if (context?.kind === "array" && context.state === "commaOrEnd") { 201 context.state = "valueOrEnd"; 202 } 203 continue; 204 } 205 206 if (char === '"') { 207 const start = state.index; 208 const end = readStringEnd(content, start); 209 const isProperty = 210 context?.kind === "object" && context.state === "keyOrEnd"; 211 pushToken( 212 state, 213 start, 214 end, 215 "string", 216 isProperty ? ["string", "property"] : ["string"], 217 ); 218 state.index = end; 219 220 if (isProperty && context.kind === "object") { 221 context.state = "colon"; 222 } else { 223 completeValue(state); 224 } 225 continue; 226 } 227 228 if (char === "-" || isDigit(char)) { 229 const end = readNumberEnd(content, state.index); 230 if (end > state.index) { 231 pushToken(state, state.index, end, "number"); 232 state.index = end; 233 completeValue(state); 234 continue; 235 } 236 } 237 238 if ( 239 consumeKeyword(state, "true", ["keyword", "constant.boolean"]) || 240 consumeKeyword(state, "false", ["keyword", "constant.boolean"]) || 241 consumeKeyword(state, "null", ["keyword", "constant.null"]) 242 ) { 243 continue; 244 } 245 246 pushToken(state, state.index, state.index + 1, "text"); 247 state.index += 1; 248 } 249 250 return { tokens: state.tokens }; 251} 252 253export const jsonLanguageSupport: ClewLanguageSupport = { 254 ids: ["json"], 255 stableBoundary: stableLineBoundary, 256 tokenize: tokenizeJson, 257};