70160
Ce
cel-tui / files
cel-tui / packages / clew / src / languages / markdown.ts
7.4 KB
1import type { ClewOutput, ClewToken } from "../model.js"; 2import type { ClewLanguageSupport } from "../registry.js"; 3import { pushTokenRange, splitLines, stableLineBoundary } from "../shared.js"; 4 5interface FenceState { 6 char: "`" | "~"; 7 length: number; 8} 9 10function pushToken( 11 tokens: ClewToken[], 12 content: string, 13 start: number, 14 end: number, 15 type: string, 16 scopes?: readonly string[], 17): void { 18 pushTokenRange(tokens, content, start, end, type, scopes); 19} 20 21function parseFence(text: string): FenceState | undefined { 22 const match = text.match(/^[ ]{0,3}([`~]{3,})(.*)$/); 23 const marker = match?.[1]; 24 if (!marker) { 25 return undefined; 26 } 27 28 const char = marker[0]; 29 if (char !== "`" && char !== "~") { 30 return undefined; 31 } 32 33 return { 34 char, 35 length: marker.length, 36 }; 37} 38 39function isFenceClose(text: string, fence: FenceState): boolean { 40 const match = text.match(/^[ ]{0,3}([`~]{3,})([ \t]*)$/); 41 const marker = match?.[1]; 42 if (!marker) { 43 return false; 44 } 45 46 return marker[0] === fence.char && marker.length >= fence.length; 47} 48 49function isHorizontalRule(text: string): boolean { 50 return /^ {0,3}(?:(?:-\s*){3,}|(?:_\s*){3,}|(?:\*\s*){3,})$/.test(text); 51} 52 53function parseHeading(text: string): 54 | { 55 bodyStart: number; 56 markerStart: number; 57 } 58 | undefined { 59 let index = 0; 60 while (index < text.length && index < 3 && text[index] === " ") { 61 index += 1; 62 } 63 64 const markerStart = index; 65 while ( 66 index < text.length && 67 index - markerStart < 6 && 68 text[index] === "#" 69 ) { 70 index += 1; 71 } 72 73 if (index === markerStart) { 74 return undefined; 75 } 76 77 const separator = text[index]; 78 if (separator !== undefined && separator !== " " && separator !== "\t") { 79 return undefined; 80 } 81 82 let bodyStart = index; 83 while (text[bodyStart] === " " || text[bodyStart] === "\t") { 84 bodyStart += 1; 85 } 86 87 return { 88 bodyStart, 89 markerStart, 90 }; 91} 92 93function readBlockquotePrefixLength(text: string): number { 94 const match = text.match(/^[ ]{0,3}(?:> ?)+/); 95 return match?.[0].length ?? 0; 96} 97 98function readListPrefixLength(text: string): number { 99 const match = text.match(/^[ ]{0,3}(?:[-+*]|\d+[.)])\s+/); 100 return match?.[0].length ?? 0; 101} 102 103function findCodeSpanEnd(text: string, start: number, count: number): number { 104 const marker = "`".repeat(count); 105 const close = text.indexOf(marker, start + count); 106 return close === -1 ? text.length : close + count; 107} 108 109function findLinkEnd(text: string, start: number): number { 110 const divider = text.indexOf("](", start + 1); 111 if (divider === -1) { 112 return -1; 113 } 114 115 const close = text.indexOf(")", divider + 2); 116 return close === -1 ? -1 : close + 1; 117} 118 119function tokenizeInline( 120 tokens: ClewToken[], 121 content: string, 122 startOffset: number, 123 text: string, 124): void { 125 let index = 0; 126 let textStart = 0; 127 128 function flush(until: number): void { 129 if (until <= textStart) { 130 return; 131 } 132 133 pushToken( 134 tokens, 135 content, 136 startOffset + textStart, 137 startOffset + until, 138 "text", 139 ); 140 textStart = until; 141 } 142 143 while (index < text.length) { 144 const char = text[index]; 145 146 if (char === "\\") { 147 flush(index); 148 const end = Math.min(index + 2, text.length); 149 pushToken( 150 tokens, 151 content, 152 startOffset + index, 153 startOffset + end, 154 "escape", 155 ); 156 index = end; 157 textStart = end; 158 continue; 159 } 160 161 if (char === "`") { 162 let count = 1; 163 while (text[index + count] === "`") { 164 count += 1; 165 } 166 167 flush(index); 168 const end = findCodeSpanEnd(text, index, count); 169 pushToken( 170 tokens, 171 content, 172 startOffset + index, 173 startOffset + end, 174 "string", 175 ["string", "markup.code"], 176 ); 177 index = end; 178 textStart = end; 179 continue; 180 } 181 182 if (char === "[") { 183 const end = findLinkEnd(text, index); 184 if (end !== -1) { 185 flush(index); 186 pushToken( 187 tokens, 188 content, 189 startOffset + index, 190 startOffset + end, 191 "link", 192 ["link", "markup.link"], 193 ); 194 index = end; 195 textStart = end; 196 continue; 197 } 198 } 199 200 index += 1; 201 } 202 203 flush(text.length); 204} 205 206function tokenizeMarkdown(content: string): ClewOutput { 207 const tokens: ClewToken[] = []; 208 const lines = splitLines(content); 209 let fence: FenceState | undefined; 210 211 for (const line of lines) { 212 const lineStart = line.start; 213 const lineEnd = line.start + line.text.length; 214 215 if (fence) { 216 if (isFenceClose(line.text, fence)) { 217 pushToken(tokens, content, lineStart, lineEnd, "meta", [ 218 "meta", 219 "markup.code.fence", 220 ]); 221 fence = undefined; 222 } else { 223 pushToken(tokens, content, lineStart, lineEnd, "string", [ 224 "string", 225 "markup.code", 226 ]); 227 } 228 } else { 229 const nextFence = parseFence(line.text); 230 if (nextFence) { 231 pushToken(tokens, content, lineStart, lineEnd, "meta", [ 232 "meta", 233 "markup.code.fence", 234 ]); 235 fence = nextFence; 236 } else if (isHorizontalRule(line.text)) { 237 pushToken(tokens, content, lineStart, lineEnd, "meta", [ 238 "meta", 239 "markup.hr", 240 ]); 241 } else { 242 const heading = parseHeading(line.text); 243 if (heading) { 244 if (heading.markerStart > 0) { 245 pushToken( 246 tokens, 247 content, 248 lineStart, 249 lineStart + heading.markerStart, 250 "whitespace", 251 ); 252 } 253 254 pushToken( 255 tokens, 256 content, 257 lineStart + heading.markerStart, 258 lineStart + heading.bodyStart, 259 "meta", 260 ["meta", "markup.heading.marker"], 261 ); 262 pushToken( 263 tokens, 264 content, 265 lineStart + heading.bodyStart, 266 lineEnd, 267 "meta", 268 ["meta", "markup.heading"], 269 ); 270 } else { 271 let inlineStart = lineStart; 272 let inlineText = line.text; 273 274 const blockquotePrefixLength = readBlockquotePrefixLength(line.text); 275 if (blockquotePrefixLength > 0) { 276 pushToken( 277 tokens, 278 content, 279 lineStart, 280 lineStart + blockquotePrefixLength, 281 "meta", 282 ["meta", "markup.quote"], 283 ); 284 inlineStart += blockquotePrefixLength; 285 inlineText = inlineText.slice(blockquotePrefixLength); 286 } else { 287 const listPrefixLength = readListPrefixLength(line.text); 288 if (listPrefixLength > 0) { 289 pushToken( 290 tokens, 291 content, 292 lineStart, 293 lineStart + listPrefixLength, 294 "meta", 295 ["meta", "markup.list"], 296 ); 297 inlineStart += listPrefixLength; 298 inlineText = inlineText.slice(listPrefixLength); 299 } 300 } 301 302 tokenizeInline(tokens, content, inlineStart, inlineText); 303 } 304 } 305 } 306 307 if (line.delimiter.length > 0) { 308 pushToken( 309 tokens, 310 content, 311 lineEnd, 312 lineEnd + line.delimiter.length, 313 "whitespace", 314 ); 315 } 316 } 317 318 return { tokens }; 319} 320 321export const markdownLanguageSupport: ClewLanguageSupport = { 322 ids: ["markdown"], 323 stableBoundary: stableLineBoundary, 324 tokenize: tokenizeMarkdown, 325};