xonecas.io
6533
Yo

yoctomarkdown / files

yoctomarkdown / src / tokenizer.ts

7.2 KB
1// Pure token-based Markdown parser — zero dependencies.
2
3export type InlineSpan =
4  | { type: "text"; content: string }
5  | { type: "bold"; content: string }
6  | { type: "italic"; content: string }
7  | { type: "code"; content: string }
8  | { type: "link"; text: string; url: string };
9
10export type BlockToken =
11  | { type: "heading"; depth: 1 | 2 | 3; spans: InlineSpan[] }
12  | { type: "paragraph"; spans: InlineSpan[] }
13  | { type: "code_block"; lang: string; content: string }
14  | { type: "list_item"; ordered: boolean; index: number; spans: InlineSpan[] }
15  | { type: "blockquote"; spans: InlineSpan[] }
16  | { type: "hr" }
17  | { type: "blank" };
18
19export interface TokenHighlighter {
20  write(chunk: string | Uint8Array): BlockToken[];
21  end(): BlockToken[];
22}
23
24// ── Inline parsing ───────────────────────────────────────────────────
25
26function parseInlineTokens(text: string): InlineSpan[] {
27  const spans: InlineSpan[] = [];
28  let i = 0;
29  let plainStart = i;
30
31  function flushPlain() {
32    if (plainStart < i) {
33      spans.push({ type: "text", content: text.slice(plainStart, i) });
34    }
35  }
36
37  while (i < text.length) {
38    // Inline code (highest priority)
39    if (text[i] === "`" && text[i + 1] !== "`") {
40      const end = text.indexOf("`", i + 1);
41      if (end !== -1) {
42        flushPlain();
43        spans.push({ type: "code", content: text.slice(i + 1, end) });
44        i = end + 1;
45        plainStart = i;
46        continue;
47      }
48    }
49
50    // Bold **text**
51    if (text[i] === "*" && text[i + 1] === "*") {
52      const end = text.indexOf("**", i + 2);
53      if (end !== -1) {
54        flushPlain();
55        spans.push({ type: "bold", content: text.slice(i + 2, end) });
56        i = end + 2;
57        plainStart = i;
58        continue;
59      }
60    }
61
62    // Italic *text* (not preceded by another *)
63    if (text[i] === "*" && text[i + 1] !== "*") {
64      const end = text.indexOf("*", i + 1);
65      if (end !== -1 && text[end + 1] !== "*") {
66        flushPlain();
67        spans.push({ type: "italic", content: text.slice(i + 1, end) });
68        i = end + 1;
69        plainStart = i;
70        continue;
71      }
72    }
73
74    // Link [text](url)
75    if (text[i] === "[") {
76      const bracketEnd = text.indexOf("]", i + 1);
77      if (bracketEnd !== -1 && text[bracketEnd + 1] === "(") {
78        const parenEnd = text.indexOf(")", bracketEnd + 2);
79        if (parenEnd !== -1) {
80          flushPlain();
81          spans.push({
82            type: "link",
83            text: text.slice(i + 1, bracketEnd),
84            url: text.slice(bracketEnd + 2, parenEnd),
85          });
86          i = parenEnd + 1;
87          plainStart = i;
88          continue;
89        }
90      }
91    }
92
93    i++;
94  }
95
96  // Remaining plain text
97  if (plainStart < text.length) {
98    spans.push({ type: "text", content: text.slice(plainStart) });
99  }
100
101  return spans;
102}
103
104// ── Block parsing ────────────────────────────────────────────────────
105
106export function parseLineToken(line: string, inCodeBlock: boolean): BlockToken {
107  if (inCodeBlock) {
108    // This function doesn't handle code block accumulation — that's
109    // done at the higher level. This shouldn't be called for lines
110    // inside a code block. Kept as a safety fallback.
111    return { type: "paragraph", spans: [{ type: "text", content: line }] };
112  }
113
114  // Fenced code block start — handled at higher level
115  if (line.trim().startsWith("```")) {
116    // Shouldn't normally reach here; the higher-level handles code blocks
117    return { type: "paragraph", spans: [{ type: "text", content: line }] };
118  }
119
120  // Horizontal rule
121  if (/^(?:-{3,}|\*{3,}|_{3,})$/.test(line.trim())) {
122    return { type: "hr" };
123  }
124
125  // Heading
126  const headingMatch = line.match(/^(#{1,3})\s+(.*)$/);
127  if (headingMatch) {
128    const depth = (headingMatch[1]?.length ?? 1) as 1 | 2 | 3;
129    const text = headingMatch[2] ?? "";
130    return { type: "heading", depth, spans: parseInlineTokens(text) };
131  }
132
133  // Blockquote
134  const bqMatch = line.match(/^>\s+(.*)$/);
135  if (bqMatch) {
136    return { type: "blockquote", spans: parseInlineTokens(bqMatch[1] ?? "") };
137  }
138
139  // Unordered list
140  const ulMatch = line.match(/^[-*]\s+(.*)$/);
141  if (ulMatch) {
142    return {
143      type: "list_item",
144      ordered: false,
145      index: 0,
146      spans: parseInlineTokens(ulMatch[1] ?? ""),
147    };
148  }
149
150  // Ordered list
151  const olMatch = line.match(/^(\d+)\.\s+(.*)$/);
152  if (olMatch) {
153    return {
154      type: "list_item",
155      ordered: true,
156      index: parseInt(olMatch[1] ?? "0", 10),
157      spans: parseInlineTokens(olMatch[2] ?? ""),
158    };
159  }
160
161  // Blank line
162  if (line.trim() === "") {
163    return { type: "blank" };
164  }
165
166  // Plain paragraph
167  return { type: "paragraph", spans: parseInlineTokens(line) };
168}
169
170// ── Batch tokenizer ─────────────────────────────────────────────────
171
172export function tokenize(input: string): BlockToken[] {
173  const tokenizer = createTokenizer();
174  const tokens = tokenizer.write(input);
175  tokens.push(...tokenizer.end());
176  return tokens;
177}
178
179// ── Streaming tokenizer ─────────────────────────────────────────────
180
181export function createTokenizer(): TokenHighlighter {
182  let buffer = "";
183  let inCodeBlock = false;
184  let codeLang = "";
185  let codeLines: string[] = [];
186
187  function processLine(line: string): BlockToken | null {
188    // Inside a code block
189    if (inCodeBlock) {
190      if (line.trim().startsWith("```")) {
191        inCodeBlock = false;
192        const token: BlockToken = {
193          type: "code_block",
194          lang: codeLang,
195          content: codeLines.join("\n"),
196        };
197        codeLang = "";
198        codeLines = [];
199        return token;
200      }
201      codeLines.push(line);
202      return null;
203    }
204
205    // Start a code block
206    if (line.trim().startsWith("```")) {
207      inCodeBlock = true;
208      const langMatch = line.trim().match(/^```(\S*)$/);
209      codeLang = langMatch?.[1] ?? "";
210      codeLines = [];
211      return null;
212    }
213
214    return parseLineToken(line, false);
215  }
216
217  return {
218    write(chunk: string | Uint8Array): BlockToken[] {
219      const text =
220        typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk);
221      buffer += text;
222
223      const lines = buffer.split("\n");
224      buffer = lines.pop() ?? "";
225
226      const tokens: BlockToken[] = [];
227      for (const line of lines) {
228        const token = processLine(line);
229        if (token) tokens.push(token);
230      }
231
232      return tokens;
233    },
234
235    end(): BlockToken[] {
236      const tokens: BlockToken[] = [];
237
238      // Flush remaining buffer
239      if (buffer.length > 0) {
240        const token = processLine(buffer);
241        if (token) tokens.push(token);
242        buffer = "";
243      }
244
245      // Flush unclosed code block
246      if (inCodeBlock) {
247        tokens.push({
248          type: "code_block",
249          lang: codeLang,
250          content: codeLines.join("\n"),
251        });
252        inCodeBlock = false;
253        codeLang = "";
254        codeLines = [];
255      }
256
257      return tokens;
258    },
259  };
260}