xonecas.io
70160
Ce

cel-tui / files

cel-tui / packages / clew / src / languages / markdown.ts

7.4 KB
1import type { ClewOutput, ClewToken } from "../model.js";
2import type { ClewLanguageSupport } from "../registry.js";
3import { pushTokenRange, splitLines, stableLineBoundary } from "../shared.js";
4
5interface FenceState {
6  char: "`" | "~";
7  length: number;
8}
9
10function pushToken(
11  tokens: ClewToken[],
12  content: string,
13  start: number,
14  end: number,
15  type: string,
16  scopes?: readonly string[],
17): void {
18  pushTokenRange(tokens, content, start, end, type, scopes);
19}
20
21function parseFence(text: string): FenceState | undefined {
22  const match = text.match(/^[ ]{0,3}([`~]{3,})(.*)$/);
23  const marker = match?.[1];
24  if (!marker) {
25    return undefined;
26  }
27
28  const char = marker[0];
29  if (char !== "`" && char !== "~") {
30    return undefined;
31  }
32
33  return {
34    char,
35    length: marker.length,
36  };
37}
38
39function isFenceClose(text: string, fence: FenceState): boolean {
40  const match = text.match(/^[ ]{0,3}([`~]{3,})([ \t]*)$/);
41  const marker = match?.[1];
42  if (!marker) {
43    return false;
44  }
45
46  return marker[0] === fence.char && marker.length >= fence.length;
47}
48
49function isHorizontalRule(text: string): boolean {
50  return /^ {0,3}(?:(?:-\s*){3,}|(?:_\s*){3,}|(?:\*\s*){3,})$/.test(text);
51}
52
53function parseHeading(text: string):
54  | {
55      bodyStart: number;
56      markerStart: number;
57    }
58  | undefined {
59  let index = 0;
60  while (index < text.length && index < 3 && text[index] === " ") {
61    index += 1;
62  }
63
64  const markerStart = index;
65  while (
66    index < text.length &&
67    index - markerStart < 6 &&
68    text[index] === "#"
69  ) {
70    index += 1;
71  }
72
73  if (index === markerStart) {
74    return undefined;
75  }
76
77  const separator = text[index];
78  if (separator !== undefined && separator !== " " && separator !== "\t") {
79    return undefined;
80  }
81
82  let bodyStart = index;
83  while (text[bodyStart] === " " || text[bodyStart] === "\t") {
84    bodyStart += 1;
85  }
86
87  return {
88    bodyStart,
89    markerStart,
90  };
91}
92
93function readBlockquotePrefixLength(text: string): number {
94  const match = text.match(/^[ ]{0,3}(?:> ?)+/);
95  return match?.[0].length ?? 0;
96}
97
98function readListPrefixLength(text: string): number {
99  const match = text.match(/^[ ]{0,3}(?:[-+*]|\d+[.)])\s+/);
100  return match?.[0].length ?? 0;
101}
102
103function findCodeSpanEnd(text: string, start: number, count: number): number {
104  const marker = "`".repeat(count);
105  const close = text.indexOf(marker, start + count);
106  return close === -1 ? text.length : close + count;
107}
108
109function findLinkEnd(text: string, start: number): number {
110  const divider = text.indexOf("](", start + 1);
111  if (divider === -1) {
112    return -1;
113  }
114
115  const close = text.indexOf(")", divider + 2);
116  return close === -1 ? -1 : close + 1;
117}
118
119function tokenizeInline(
120  tokens: ClewToken[],
121  content: string,
122  startOffset: number,
123  text: string,
124): void {
125  let index = 0;
126  let textStart = 0;
127
128  function flush(until: number): void {
129    if (until <= textStart) {
130      return;
131    }
132
133    pushToken(
134      tokens,
135      content,
136      startOffset + textStart,
137      startOffset + until,
138      "text",
139    );
140    textStart = until;
141  }
142
143  while (index < text.length) {
144    const char = text[index];
145
146    if (char === "\\") {
147      flush(index);
148      const end = Math.min(index + 2, text.length);
149      pushToken(
150        tokens,
151        content,
152        startOffset + index,
153        startOffset + end,
154        "escape",
155      );
156      index = end;
157      textStart = end;
158      continue;
159    }
160
161    if (char === "`") {
162      let count = 1;
163      while (text[index + count] === "`") {
164        count += 1;
165      }
166
167      flush(index);
168      const end = findCodeSpanEnd(text, index, count);
169      pushToken(
170        tokens,
171        content,
172        startOffset + index,
173        startOffset + end,
174        "string",
175        ["string", "markup.code"],
176      );
177      index = end;
178      textStart = end;
179      continue;
180    }
181
182    if (char === "[") {
183      const end = findLinkEnd(text, index);
184      if (end !== -1) {
185        flush(index);
186        pushToken(
187          tokens,
188          content,
189          startOffset + index,
190          startOffset + end,
191          "link",
192          ["link", "markup.link"],
193        );
194        index = end;
195        textStart = end;
196        continue;
197      }
198    }
199
200    index += 1;
201  }
202
203  flush(text.length);
204}
205
206function tokenizeMarkdown(content: string): ClewOutput {
207  const tokens: ClewToken[] = [];
208  const lines = splitLines(content);
209  let fence: FenceState | undefined;
210
211  for (const line of lines) {
212    const lineStart = line.start;
213    const lineEnd = line.start + line.text.length;
214
215    if (fence) {
216      if (isFenceClose(line.text, fence)) {
217        pushToken(tokens, content, lineStart, lineEnd, "meta", [
218          "meta",
219          "markup.code.fence",
220        ]);
221        fence = undefined;
222      } else {
223        pushToken(tokens, content, lineStart, lineEnd, "string", [
224          "string",
225          "markup.code",
226        ]);
227      }
228    } else {
229      const nextFence = parseFence(line.text);
230      if (nextFence) {
231        pushToken(tokens, content, lineStart, lineEnd, "meta", [
232          "meta",
233          "markup.code.fence",
234        ]);
235        fence = nextFence;
236      } else if (isHorizontalRule(line.text)) {
237        pushToken(tokens, content, lineStart, lineEnd, "meta", [
238          "meta",
239          "markup.hr",
240        ]);
241      } else {
242        const heading = parseHeading(line.text);
243        if (heading) {
244          if (heading.markerStart > 0) {
245            pushToken(
246              tokens,
247              content,
248              lineStart,
249              lineStart + heading.markerStart,
250              "whitespace",
251            );
252          }
253
254          pushToken(
255            tokens,
256            content,
257            lineStart + heading.markerStart,
258            lineStart + heading.bodyStart,
259            "meta",
260            ["meta", "markup.heading.marker"],
261          );
262          pushToken(
263            tokens,
264            content,
265            lineStart + heading.bodyStart,
266            lineEnd,
267            "meta",
268            ["meta", "markup.heading"],
269          );
270        } else {
271          let inlineStart = lineStart;
272          let inlineText = line.text;
273
274          const blockquotePrefixLength = readBlockquotePrefixLength(line.text);
275          if (blockquotePrefixLength > 0) {
276            pushToken(
277              tokens,
278              content,
279              lineStart,
280              lineStart + blockquotePrefixLength,
281              "meta",
282              ["meta", "markup.quote"],
283            );
284            inlineStart += blockquotePrefixLength;
285            inlineText = inlineText.slice(blockquotePrefixLength);
286          } else {
287            const listPrefixLength = readListPrefixLength(line.text);
288            if (listPrefixLength > 0) {
289              pushToken(
290                tokens,
291                content,
292                lineStart,
293                lineStart + listPrefixLength,
294                "meta",
295                ["meta", "markup.list"],
296              );
297              inlineStart += listPrefixLength;
298              inlineText = inlineText.slice(listPrefixLength);
299            }
300          }
301
302          tokenizeInline(tokens, content, inlineStart, inlineText);
303        }
304      }
305    }
306
307    if (line.delimiter.length > 0) {
308      pushToken(
309        tokens,
310        content,
311        lineEnd,
312        lineEnd + line.delimiter.length,
313        "whitespace",
314      );
315    }
316  }
317
318  return { tokens };
319}
320
321export const markdownLanguageSupport: ClewLanguageSupport = {
322  ids: ["markdown"],
323  stableBoundary: stableLineBoundary,
324  tokenize: tokenizeMarkdown,
325};