6533
Yo
yoctomarkdown / files
yoctomarkdown / src / tokenizer.ts
7.2 KB
1// Pure token-based Markdown parser — zero dependencies. 2 3export type InlineSpan = 4 | { type: "text"; content: string } 5 | { type: "bold"; content: string } 6 | { type: "italic"; content: string } 7 | { type: "code"; content: string } 8 | { type: "link"; text: string; url: string }; 9 10export type BlockToken = 11 | { type: "heading"; depth: 1 | 2 | 3; spans: InlineSpan[] } 12 | { type: "paragraph"; spans: InlineSpan[] } 13 | { type: "code_block"; lang: string; content: string } 14 | { type: "list_item"; ordered: boolean; index: number; spans: InlineSpan[] } 15 | { type: "blockquote"; spans: InlineSpan[] } 16 | { type: "hr" } 17 | { type: "blank" }; 18 19export interface TokenHighlighter { 20 write(chunk: string | Uint8Array): BlockToken[]; 21 end(): BlockToken[]; 22} 23 24// ── Inline parsing ─────────────────────────────────────────────────── 25 26function parseInlineTokens(text: string): InlineSpan[] { 27 const spans: InlineSpan[] = []; 28 let i = 0; 29 let plainStart = i; 30 31 function flushPlain() { 32 if (plainStart < i) { 33 spans.push({ type: "text", content: text.slice(plainStart, i) }); 34 } 35 } 36 37 while (i < text.length) { 38 // Inline code (highest priority) 39 if (text[i] === "`" && text[i + 1] !== "`") { 40 const end = text.indexOf("`", i + 1); 41 if (end !== -1) { 42 flushPlain(); 43 spans.push({ type: "code", content: text.slice(i + 1, end) }); 44 i = end + 1; 45 plainStart = i; 46 continue; 47 } 48 } 49 50 // Bold **text** 51 if (text[i] === "*" && text[i + 1] === "*") { 52 const end = text.indexOf("**", i + 2); 53 if (end !== -1) { 54 flushPlain(); 55 spans.push({ type: "bold", content: text.slice(i + 2, end) }); 56 i = end + 2; 57 plainStart = i; 58 continue; 59 } 60 } 61 62 // Italic *text* (not preceded by another *) 63 if (text[i] === "*" && text[i + 1] !== "*") { 64 const end = text.indexOf("*", i + 1); 65 if (end !== -1 && text[end + 1] !== "*") { 66 flushPlain(); 67 spans.push({ type: "italic", content: text.slice(i + 1, end) }); 68 i = end + 1; 69 plainStart = i; 70 continue; 71 } 72 } 73 74 // Link [text](url) 75 if (text[i] === "[") { 76 const bracketEnd = text.indexOf("]", i + 1); 77 if (bracketEnd !== -1 && text[bracketEnd + 1] === "(") { 78 const parenEnd = text.indexOf(")", bracketEnd + 2); 79 if (parenEnd !== -1) { 80 flushPlain(); 81 spans.push({ 82 type: "link", 83 text: text.slice(i + 1, bracketEnd), 84 url: text.slice(bracketEnd + 2, parenEnd), 85 }); 86 i = parenEnd + 1; 87 plainStart = i; 88 continue; 89 } 90 } 91 } 92 93 i++; 94 } 95 96 // Remaining plain text 97 if (plainStart < text.length) { 98 spans.push({ type: "text", content: text.slice(plainStart) }); 99 } 100 101 return spans; 102} 103 104// ── Block parsing ──────────────────────────────────────────────────── 105 106export function parseLineToken(line: string, inCodeBlock: boolean): BlockToken { 107 if (inCodeBlock) { 108 // This function doesn't handle code block accumulation — that's 109 // done at the higher level. This shouldn't be called for lines 110 // inside a code block. Kept as a safety fallback. 111 return { type: "paragraph", spans: [{ type: "text", content: line }] }; 112 } 113 114 // Fenced code block start — handled at higher level 115 if (line.trim().startsWith("```")) { 116 // Shouldn't normally reach here; the higher-level handles code blocks 117 return { type: "paragraph", spans: [{ type: "text", content: line }] }; 118 } 119 120 // Horizontal rule 121 if (/^(?:-{3,}|\*{3,}|_{3,})$/.test(line.trim())) { 122 return { type: "hr" }; 123 } 124 125 // Heading 126 const headingMatch = line.match(/^(#{1,3})\s+(.*)$/); 127 if (headingMatch) { 128 const depth = (headingMatch[1]?.length ?? 1) as 1 | 2 | 3; 129 const text = headingMatch[2] ?? ""; 130 return { type: "heading", depth, spans: parseInlineTokens(text) }; 131 } 132 133 // Blockquote 134 const bqMatch = line.match(/^>\s+(.*)$/); 135 if (bqMatch) { 136 return { type: "blockquote", spans: parseInlineTokens(bqMatch[1] ?? "") }; 137 } 138 139 // Unordered list 140 const ulMatch = line.match(/^[-*]\s+(.*)$/); 141 if (ulMatch) { 142 return { 143 type: "list_item", 144 ordered: false, 145 index: 0, 146 spans: parseInlineTokens(ulMatch[1] ?? ""), 147 }; 148 } 149 150 // Ordered list 151 const olMatch = line.match(/^(\d+)\.\s+(.*)$/); 152 if (olMatch) { 153 return { 154 type: "list_item", 155 ordered: true, 156 index: parseInt(olMatch[1] ?? "0", 10), 157 spans: parseInlineTokens(olMatch[2] ?? ""), 158 }; 159 } 160 161 // Blank line 162 if (line.trim() === "") { 163 return { type: "blank" }; 164 } 165 166 // Plain paragraph 167 return { type: "paragraph", spans: parseInlineTokens(line) }; 168} 169 170// ── Batch tokenizer ───────────────────────────────────────────────── 171 172export function tokenize(input: string): BlockToken[] { 173 const tokenizer = createTokenizer(); 174 const tokens = tokenizer.write(input); 175 tokens.push(...tokenizer.end()); 176 return tokens; 177} 178 179// ── Streaming tokenizer ───────────────────────────────────────────── 180 181export function createTokenizer(): TokenHighlighter { 182 let buffer = ""; 183 let inCodeBlock = false; 184 let codeLang = ""; 185 let codeLines: string[] = []; 186 187 function processLine(line: string): BlockToken | null { 188 // Inside a code block 189 if (inCodeBlock) { 190 if (line.trim().startsWith("```")) { 191 inCodeBlock = false; 192 const token: BlockToken = { 193 type: "code_block", 194 lang: codeLang, 195 content: codeLines.join("\n"), 196 }; 197 codeLang = ""; 198 codeLines = []; 199 return token; 200 } 201 codeLines.push(line); 202 return null; 203 } 204 205 // Start a code block 206 if (line.trim().startsWith("```")) { 207 inCodeBlock = true; 208 const langMatch = line.trim().match(/^```(\S*)$/); 209 codeLang = langMatch?.[1] ?? ""; 210 codeLines = []; 211 return null; 212 } 213 214 return parseLineToken(line, false); 215 } 216 217 return { 218 write(chunk: string | Uint8Array): BlockToken[] { 219 const text = 220 typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk); 221 buffer += text; 222 223 const lines = buffer.split("\n"); 224 buffer = lines.pop() ?? ""; 225 226 const tokens: BlockToken[] = []; 227 for (const line of lines) { 228 const token = processLine(line); 229 if (token) tokens.push(token); 230 } 231 232 return tokens; 233 }, 234 235 end(): BlockToken[] { 236 const tokens: BlockToken[] = []; 237 238 // Flush remaining buffer 239 if (buffer.length > 0) { 240 const token = processLine(buffer); 241 if (token) tokens.push(token); 242 buffer = ""; 243 } 244 245 // Flush unclosed code block 246 if (inCodeBlock) { 247 tokens.push({ 248 type: "code_block", 249 lang: codeLang, 250 content: codeLines.join("\n"), 251 }); 252 inCodeBlock = false; 253 codeLang = ""; 254 codeLines = []; 255 } 256 257 return tokens; 258 }, 259 }; 260}