6533
Yo
yoctomarkdown / files
yoctomarkdown / tests / tokenizer.test.ts
13.8 KB
1import { test, expect, describe } from "bun:test"; 2import { tokenize, createTokenizer, type BlockToken } from "../src/index"; 3 4// ── Inline span parsing ────────────────────────────────────────────── 5 6describe("tokenize: inline spans", () => { 7 test("plain text", () => { 8 const tokens = tokenize("Hello world"); 9 expect(tokens).toEqual([ 10 { type: "paragraph", spans: [{ type: "text", content: "Hello world" }] }, 11 ]); 12 }); 13 14 test("bold", () => { 15 const tokens = tokenize("some **bold** text"); 16 expect(tokens).toEqual([ 17 { 18 type: "paragraph", 19 spans: [ 20 { type: "text", content: "some " }, 21 { type: "bold", content: "bold" }, 22 { type: "text", content: " text" }, 23 ], 24 }, 25 ]); 26 }); 27 28 test("italic", () => { 29 const tokens = tokenize("some *italic* text"); 30 expect(tokens).toEqual([ 31 { 32 type: "paragraph", 33 spans: [ 34 { type: "text", content: "some " }, 35 { type: "italic", content: "italic" }, 36 { type: "text", content: " text" }, 37 ], 38 }, 39 ]); 40 }); 41 42 test("inline code", () => { 43 const tokens = tokenize("use `foo()` here"); 44 expect(tokens).toEqual([ 45 { 46 type: "paragraph", 47 spans: [ 48 { type: "text", content: "use " }, 49 { type: "code", content: "foo()" }, 50 { type: "text", content: " here" }, 51 ], 52 }, 53 ]); 54 }); 55 56 test("link", () => { 57 const tokens = tokenize("click [here](https://example.com) now"); 58 expect(tokens).toEqual([ 59 { 60 type: "paragraph", 61 spans: [ 62 { type: "text", content: "click " }, 63 { type: "link", text: "here", url: "https://example.com" }, 64 { type: "text", content: " now" }, 65 ], 66 }, 67 ]); 68 }); 69 70 test("mixed inline formatting", () => { 71 const tokens = tokenize("**bold** and *italic* and `code`"); 72 expect(tokens).toEqual([ 73 { 74 type: "paragraph", 75 spans: [ 76 { type: "bold", content: "bold" }, 77 { type: "text", content: " and " }, 78 { type: "italic", content: "italic" }, 79 { type: "text", content: " and " }, 80 { type: "code", content: "code" }, 81 ], 82 }, 83 ]); 84 }); 85 86 test("unclosed bold falls back to text", () => { 87 const tokens = tokenize("some **unclosed bold"); 88 expect(tokens).toEqual([ 89 { 90 type: "paragraph", 91 spans: [{ type: "text", content: "some **unclosed bold" }], 92 }, 93 ]); 94 }); 95 96 test("unclosed italic falls back to text", () => { 97 const tokens = tokenize("some *unclosed italic"); 98 expect(tokens).toEqual([ 99 { 100 type: "paragraph", 101 spans: [{ type: "text", content: "some *unclosed italic" }], 102 }, 103 ]); 104 }); 105 106 test("unclosed inline code falls back to text", () => { 107 const tokens = tokenize("some `unclosed code"); 108 expect(tokens).toEqual([ 109 { 110 type: "paragraph", 111 spans: [{ type: "text", content: "some `unclosed code" }], 112 }, 113 ]); 114 }); 115 116 test("inline code takes priority over bold", () => { 117 const tokens = tokenize("`**not bold**`"); 118 expect(tokens).toEqual([ 119 { 120 type: "paragraph", 121 spans: [{ type: "code", content: "**not bold**" }], 122 }, 123 ]); 124 }); 125}); 126 127// ── Block-level tokens ─────────────────────────────────────────────── 128 129describe("tokenize: block tokens", () => { 130 test("heading 1", () => { 131 const tokens = tokenize("# Hello"); 132 expect(tokens).toEqual([ 133 { 134 type: "heading", 135 depth: 1, 136 spans: [{ type: "text", content: "Hello" }], 137 }, 138 ]); 139 }); 140 141 test("heading 2", () => { 142 const tokens = tokenize("## Sub"); 143 expect(tokens).toEqual([ 144 { 145 type: "heading", 146 depth: 2, 147 spans: [{ type: "text", content: "Sub" }], 148 }, 149 ]); 150 }); 151 152 test("heading 3", () => { 153 const tokens = tokenize("### Third"); 154 expect(tokens).toEqual([ 155 { 156 type: "heading", 157 depth: 3, 158 spans: [{ type: "text", content: "Third" }], 159 }, 160 ]); 161 }); 162 163 test("heading with inline formatting", () => { 164 const tokens = tokenize("# **Bold** heading"); 165 expect(tokens).toEqual([ 166 { 167 type: "heading", 168 depth: 1, 169 spans: [ 170 { type: "bold", content: "Bold" }, 171 { type: "text", content: " heading" }, 172 ], 173 }, 174 ]); 175 }); 176 177 test("fenced code block", () => { 178 const tokens = tokenize("```js\nconst x = 1;\n```"); 179 expect(tokens).toEqual([ 180 { type: "code_block", lang: "js", content: "const x = 1;" }, 181 ]); 182 }); 183 184 test("fenced code block without language", () => { 185 const tokens = tokenize("```\nhello\n```"); 186 expect(tokens).toEqual([ 187 { type: "code_block", lang: "", content: "hello" }, 188 ]); 189 }); 190 191 test("fenced code block with multiple lines", () => { 192 const tokens = tokenize("```\nline1\nline2\nline3\n```"); 193 expect(tokens).toEqual([ 194 { type: "code_block", lang: "", content: "line1\nline2\nline3" }, 195 ]); 196 }); 197 198 test("unclosed code block emits as code_block on end", () => { 199 const tokens = tokenize("```\nunclosed code"); 200 expect(tokens).toEqual([ 201 { type: "code_block", lang: "", content: "unclosed code" }, 202 ]); 203 }); 204 205 test("unordered list item with dash", () => { 206 const tokens = tokenize("- Item one"); 207 expect(tokens).toEqual([ 208 { 209 type: "list_item", 210 ordered: false, 211 index: 0, 212 spans: [{ type: "text", content: "Item one" }], 213 }, 214 ]); 215 }); 216 217 test("unordered list item with asterisk", () => { 218 const tokens = tokenize("* Item two"); 219 expect(tokens).toEqual([ 220 { 221 type: "list_item", 222 ordered: false, 223 index: 0, 224 spans: [{ type: "text", content: "Item two" }], 225 }, 226 ]); 227 }); 228 229 test("ordered list item", () => { 230 const tokens = tokenize("1. First"); 231 expect(tokens).toEqual([ 232 { 233 type: "list_item", 234 ordered: true, 235 index: 1, 236 spans: [{ type: "text", content: "First" }], 237 }, 238 ]); 239 }); 240 241 test("ordered list preserves index", () => { 242 const tokens = tokenize("3. Third"); 243 expect(tokens).toEqual([ 244 { 245 type: "list_item", 246 ordered: true, 247 index: 3, 248 spans: [{ type: "text", content: "Third" }], 249 }, 250 ]); 251 }); 252 253 test("blockquote", () => { 254 const tokens = tokenize("> quoted text"); 255 expect(tokens).toEqual([ 256 { 257 type: "blockquote", 258 spans: [{ type: "text", content: "quoted text" }], 259 }, 260 ]); 261 }); 262 263 test("blockquote with inline formatting", () => { 264 const tokens = tokenize("> **bold** quote"); 265 expect(tokens).toEqual([ 266 { 267 type: "blockquote", 268 spans: [ 269 { type: "bold", content: "bold" }, 270 { type: "text", content: " quote" }, 271 ], 272 }, 273 ]); 274 }); 275 276 test("horizontal rule ---", () => { 277 const tokens = tokenize("---"); 278 expect(tokens).toEqual([{ type: "hr" }]); 279 }); 280 281 test("horizontal rule ***", () => { 282 const tokens = tokenize("***"); 283 expect(tokens).toEqual([{ type: "hr" }]); 284 }); 285 286 test("horizontal rule ___", () => { 287 const tokens = tokenize("___"); 288 expect(tokens).toEqual([{ type: "hr" }]); 289 }); 290 291 test("blank line", () => { 292 const tokens = tokenize("hello\n\nworld"); 293 expect(tokens).toEqual([ 294 { type: "paragraph", spans: [{ type: "text", content: "hello" }] }, 295 { type: "blank" }, 296 { type: "paragraph", spans: [{ type: "text", content: "world" }] }, 297 ]); 298 }); 299}); 300 301// ── Full document tokenization ─────────────────────────────────────── 302 303describe("tokenize: full documents", () => { 304 test("mixed content document", () => { 305 const input = [ 306 "# Title", 307 "", 308 "Some **bold** and *italic* text.", 309 "", 310 "```js", 311 "const x = 1;", 312 "```", 313 "", 314 "- item 1", 315 "- item 2", 316 "", 317 "> a quote", 318 "", 319 "---", 320 ].join("\n"); 321 322 const tokens = tokenize(input); 323 expect(tokens[0]).toEqual({ 324 type: "heading", 325 depth: 1, 326 spans: [{ type: "text", content: "Title" }], 327 }); 328 expect(tokens[1]).toEqual({ type: "blank" }); 329 expect(tokens[2]).toMatchObject({ type: "paragraph" }); 330 expect(tokens[3]).toEqual({ type: "blank" }); 331 expect(tokens[4]).toEqual({ 332 type: "code_block", 333 lang: "js", 334 content: "const x = 1;", 335 }); 336 expect(tokens[5]).toEqual({ type: "blank" }); 337 expect(tokens[6]).toMatchObject({ 338 type: "list_item", 339 ordered: false, 340 index: 0, 341 }); 342 expect(tokens[7]).toMatchObject({ 343 type: "list_item", 344 ordered: false, 345 index: 0, 346 }); 347 expect(tokens[8]).toEqual({ type: "blank" }); 348 expect(tokens[9]).toMatchObject({ type: "blockquote" }); 349 expect(tokens[10]).toEqual({ type: "blank" }); 350 expect(tokens[11]).toEqual({ type: "hr" }); 351 }); 352}); 353 354// ── Streaming tokenizer ───────────────────────────────────────────── 355 356describe("createTokenizer: streaming", () => { 357 test("basic streaming produces same tokens as batch", () => { 358 const input = "# Hello\n\nSome **bold** text.\n"; 359 const batchTokens = tokenize(input); 360 361 const tokenizer = createTokenizer(); 362 const streamTokens: BlockToken[] = []; 363 streamTokens.push(...tokenizer.write(input)); 364 streamTokens.push(...tokenizer.end()); 365 366 expect(streamTokens).toEqual(batchTokens); 367 }); 368 369 test("character-by-character streaming", () => { 370 const input = "# Title\n\n- item 1\n- item 2\n"; 371 const batchTokens = tokenize(input); 372 373 const tokenizer = createTokenizer(); 374 const streamTokens: BlockToken[] = []; 375 for (const ch of input) { 376 streamTokens.push(...tokenizer.write(ch)); 377 } 378 streamTokens.push(...tokenizer.end()); 379 380 expect(streamTokens).toEqual(batchTokens); 381 }); 382 383 test("chunks split across line boundaries", () => { 384 const tokenizer = createTokenizer(); 385 const tokens: BlockToken[] = []; 386 387 tokens.push(...tokenizer.write("# Hea")); 388 tokens.push(...tokenizer.write("ding\n")); 389 tokens.push(...tokenizer.write("\nSome text\n")); 390 tokens.push(...tokenizer.end()); 391 392 expect(tokens[0]).toEqual({ 393 type: "heading", 394 depth: 1, 395 spans: [{ type: "text", content: "Heading" }], 396 }); 397 expect(tokens[1]).toEqual({ type: "blank" }); 398 expect(tokens[2]).toEqual({ 399 type: "paragraph", 400 spans: [{ type: "text", content: "Some text" }], 401 }); 402 }); 403 404 test("emits completed tokens immediately", () => { 405 const tokenizer = createTokenizer(); 406 407 // Incomplete line — no token emitted yet 408 const t1 = tokenizer.write("# Hel"); 409 expect(t1).toEqual([]); 410 411 // Complete the line — token emitted 412 const t2 = tokenizer.write("lo\n"); 413 expect(t2).toEqual([ 414 { 415 type: "heading", 416 depth: 1, 417 spans: [{ type: "text", content: "Hello" }], 418 }, 419 ]); 420 }); 421 422 test("buffers only incomplete line", () => { 423 const tokenizer = createTokenizer(); 424 425 const t1 = tokenizer.write("hello\nwor"); 426 expect(t1).toEqual([ 427 { type: "paragraph", spans: [{ type: "text", content: "hello" }] }, 428 ]); 429 430 // "wor" is buffered, flushed on end() 431 const t2 = tokenizer.end(); 432 expect(t2).toEqual([ 433 { type: "paragraph", spans: [{ type: "text", content: "wor" }] }, 434 ]); 435 }); 436 437 test("code block spanning multiple writes", () => { 438 const tokenizer = createTokenizer(); 439 const tokens: BlockToken[] = []; 440 441 tokens.push(...tokenizer.write("```js\ncon")); 442 tokens.push(...tokenizer.write("st x = 1;\n```\n")); 443 tokens.push(...tokenizer.end()); 444 445 expect(tokens).toContainEqual({ 446 type: "code_block", 447 lang: "js", 448 content: "const x = 1;", 449 }); 450 }); 451 452 test("unclosed code block flushed on end()", () => { 453 const tokenizer = createTokenizer(); 454 const tokens: BlockToken[] = []; 455 456 tokens.push(...tokenizer.write("```\nsome code")); 457 tokens.push(...tokenizer.end()); 458 459 expect(tokens).toContainEqual({ 460 type: "code_block", 461 lang: "", 462 content: "some code", 463 }); 464 }); 465 466 test("accepts Uint8Array chunks", () => { 467 const tokenizer = createTokenizer(); 468 const tokens: BlockToken[] = []; 469 470 tokens.push(...tokenizer.write(new TextEncoder().encode("# Hello\n"))); 471 tokens.push(...tokenizer.end()); 472 473 expect(tokens).toEqual([ 474 { 475 type: "heading", 476 depth: 1, 477 spans: [{ type: "text", content: "Hello" }], 478 }, 479 ]); 480 }); 481 482 test("random chunk sizes match batch", () => { 483 const input = 484 "```javascript\nconst x = 1;\n```\n\n# Heading 2\n\nSome text with `code`.\n> Blockquote here\n\n1. One\n2. Two\n"; 485 const batchTokens = tokenize(input); 486 487 const tokenizer = createTokenizer(); 488 const streamTokens: BlockToken[] = []; 489 490 let i = 0; 491 while (i < input.length) { 492 const chunkSize = Math.floor(Math.random() * 10) + 1; 493 streamTokens.push(...tokenizer.write(input.slice(i, i + chunkSize))); 494 i += chunkSize; 495 } 496 streamTokens.push(...tokenizer.end()); 497 498 expect(streamTokens).toEqual(batchTokens); 499 }); 500}); 501 502// ── Performance ────────────────────────────────────────────────────── 503 504describe("tokenizer: performance", () => { 505 test("batch tokenize 5000 blocks under 200ms", () => { 506 const largeMd = "# Heading\n\nSome **bold** text here.\n".repeat(5000); 507 const start = performance.now(); 508 tokenize(largeMd); 509 const duration = performance.now() - start; 510 console.log(`Token batch parse time: ${duration.toFixed(2)}ms`); 511 expect(duration).toBeLessThan(200); 512 }); 513});