xonecas.io
6533
Yo

yoctomarkdown / files

yoctomarkdown / tests / tokenizer.test.ts

13.8 KB
1import { test, expect, describe } from "bun:test";
2import { tokenize, createTokenizer, type BlockToken } from "../src/index";
3
4// ── Inline span parsing ──────────────────────────────────────────────
5
6describe("tokenize: inline spans", () => {
7  test("plain text", () => {
8    const tokens = tokenize("Hello world");
9    expect(tokens).toEqual([
10      { type: "paragraph", spans: [{ type: "text", content: "Hello world" }] },
11    ]);
12  });
13
14  test("bold", () => {
15    const tokens = tokenize("some **bold** text");
16    expect(tokens).toEqual([
17      {
18        type: "paragraph",
19        spans: [
20          { type: "text", content: "some " },
21          { type: "bold", content: "bold" },
22          { type: "text", content: " text" },
23        ],
24      },
25    ]);
26  });
27
28  test("italic", () => {
29    const tokens = tokenize("some *italic* text");
30    expect(tokens).toEqual([
31      {
32        type: "paragraph",
33        spans: [
34          { type: "text", content: "some " },
35          { type: "italic", content: "italic" },
36          { type: "text", content: " text" },
37        ],
38      },
39    ]);
40  });
41
42  test("inline code", () => {
43    const tokens = tokenize("use `foo()` here");
44    expect(tokens).toEqual([
45      {
46        type: "paragraph",
47        spans: [
48          { type: "text", content: "use " },
49          { type: "code", content: "foo()" },
50          { type: "text", content: " here" },
51        ],
52      },
53    ]);
54  });
55
56  test("link", () => {
57    const tokens = tokenize("click [here](https://example.com) now");
58    expect(tokens).toEqual([
59      {
60        type: "paragraph",
61        spans: [
62          { type: "text", content: "click " },
63          { type: "link", text: "here", url: "https://example.com" },
64          { type: "text", content: " now" },
65        ],
66      },
67    ]);
68  });
69
70  test("mixed inline formatting", () => {
71    const tokens = tokenize("**bold** and *italic* and `code`");
72    expect(tokens).toEqual([
73      {
74        type: "paragraph",
75        spans: [
76          { type: "bold", content: "bold" },
77          { type: "text", content: " and " },
78          { type: "italic", content: "italic" },
79          { type: "text", content: " and " },
80          { type: "code", content: "code" },
81        ],
82      },
83    ]);
84  });
85
86  test("unclosed bold falls back to text", () => {
87    const tokens = tokenize("some **unclosed bold");
88    expect(tokens).toEqual([
89      {
90        type: "paragraph",
91        spans: [{ type: "text", content: "some **unclosed bold" }],
92      },
93    ]);
94  });
95
96  test("unclosed italic falls back to text", () => {
97    const tokens = tokenize("some *unclosed italic");
98    expect(tokens).toEqual([
99      {
100        type: "paragraph",
101        spans: [{ type: "text", content: "some *unclosed italic" }],
102      },
103    ]);
104  });
105
106  test("unclosed inline code falls back to text", () => {
107    const tokens = tokenize("some `unclosed code");
108    expect(tokens).toEqual([
109      {
110        type: "paragraph",
111        spans: [{ type: "text", content: "some `unclosed code" }],
112      },
113    ]);
114  });
115
116  test("inline code takes priority over bold", () => {
117    const tokens = tokenize("`**not bold**`");
118    expect(tokens).toEqual([
119      {
120        type: "paragraph",
121        spans: [{ type: "code", content: "**not bold**" }],
122      },
123    ]);
124  });
125});
126
127// ── Block-level tokens ───────────────────────────────────────────────
128
129describe("tokenize: block tokens", () => {
130  test("heading 1", () => {
131    const tokens = tokenize("# Hello");
132    expect(tokens).toEqual([
133      {
134        type: "heading",
135        depth: 1,
136        spans: [{ type: "text", content: "Hello" }],
137      },
138    ]);
139  });
140
141  test("heading 2", () => {
142    const tokens = tokenize("## Sub");
143    expect(tokens).toEqual([
144      {
145        type: "heading",
146        depth: 2,
147        spans: [{ type: "text", content: "Sub" }],
148      },
149    ]);
150  });
151
152  test("heading 3", () => {
153    const tokens = tokenize("### Third");
154    expect(tokens).toEqual([
155      {
156        type: "heading",
157        depth: 3,
158        spans: [{ type: "text", content: "Third" }],
159      },
160    ]);
161  });
162
163  test("heading with inline formatting", () => {
164    const tokens = tokenize("# **Bold** heading");
165    expect(tokens).toEqual([
166      {
167        type: "heading",
168        depth: 1,
169        spans: [
170          { type: "bold", content: "Bold" },
171          { type: "text", content: " heading" },
172        ],
173      },
174    ]);
175  });
176
177  test("fenced code block", () => {
178    const tokens = tokenize("```js\nconst x = 1;\n```");
179    expect(tokens).toEqual([
180      { type: "code_block", lang: "js", content: "const x = 1;" },
181    ]);
182  });
183
184  test("fenced code block without language", () => {
185    const tokens = tokenize("```\nhello\n```");
186    expect(tokens).toEqual([
187      { type: "code_block", lang: "", content: "hello" },
188    ]);
189  });
190
191  test("fenced code block with multiple lines", () => {
192    const tokens = tokenize("```\nline1\nline2\nline3\n```");
193    expect(tokens).toEqual([
194      { type: "code_block", lang: "", content: "line1\nline2\nline3" },
195    ]);
196  });
197
198  test("unclosed code block emits as code_block on end", () => {
199    const tokens = tokenize("```\nunclosed code");
200    expect(tokens).toEqual([
201      { type: "code_block", lang: "", content: "unclosed code" },
202    ]);
203  });
204
205  test("unordered list item with dash", () => {
206    const tokens = tokenize("- Item one");
207    expect(tokens).toEqual([
208      {
209        type: "list_item",
210        ordered: false,
211        index: 0,
212        spans: [{ type: "text", content: "Item one" }],
213      },
214    ]);
215  });
216
217  test("unordered list item with asterisk", () => {
218    const tokens = tokenize("* Item two");
219    expect(tokens).toEqual([
220      {
221        type: "list_item",
222        ordered: false,
223        index: 0,
224        spans: [{ type: "text", content: "Item two" }],
225      },
226    ]);
227  });
228
229  test("ordered list item", () => {
230    const tokens = tokenize("1. First");
231    expect(tokens).toEqual([
232      {
233        type: "list_item",
234        ordered: true,
235        index: 1,
236        spans: [{ type: "text", content: "First" }],
237      },
238    ]);
239  });
240
241  test("ordered list preserves index", () => {
242    const tokens = tokenize("3. Third");
243    expect(tokens).toEqual([
244      {
245        type: "list_item",
246        ordered: true,
247        index: 3,
248        spans: [{ type: "text", content: "Third" }],
249      },
250    ]);
251  });
252
253  test("blockquote", () => {
254    const tokens = tokenize("> quoted text");
255    expect(tokens).toEqual([
256      {
257        type: "blockquote",
258        spans: [{ type: "text", content: "quoted text" }],
259      },
260    ]);
261  });
262
263  test("blockquote with inline formatting", () => {
264    const tokens = tokenize("> **bold** quote");
265    expect(tokens).toEqual([
266      {
267        type: "blockquote",
268        spans: [
269          { type: "bold", content: "bold" },
270          { type: "text", content: " quote" },
271        ],
272      },
273    ]);
274  });
275
276  test("horizontal rule ---", () => {
277    const tokens = tokenize("---");
278    expect(tokens).toEqual([{ type: "hr" }]);
279  });
280
281  test("horizontal rule ***", () => {
282    const tokens = tokenize("***");
283    expect(tokens).toEqual([{ type: "hr" }]);
284  });
285
286  test("horizontal rule ___", () => {
287    const tokens = tokenize("___");
288    expect(tokens).toEqual([{ type: "hr" }]);
289  });
290
291  test("blank line", () => {
292    const tokens = tokenize("hello\n\nworld");
293    expect(tokens).toEqual([
294      { type: "paragraph", spans: [{ type: "text", content: "hello" }] },
295      { type: "blank" },
296      { type: "paragraph", spans: [{ type: "text", content: "world" }] },
297    ]);
298  });
299});
300
301// ── Full document tokenization ───────────────────────────────────────
302
303describe("tokenize: full documents", () => {
304  test("mixed content document", () => {
305    const input = [
306      "# Title",
307      "",
308      "Some **bold** and *italic* text.",
309      "",
310      "```js",
311      "const x = 1;",
312      "```",
313      "",
314      "- item 1",
315      "- item 2",
316      "",
317      "> a quote",
318      "",
319      "---",
320    ].join("\n");
321
322    const tokens = tokenize(input);
323    expect(tokens[0]).toEqual({
324      type: "heading",
325      depth: 1,
326      spans: [{ type: "text", content: "Title" }],
327    });
328    expect(tokens[1]).toEqual({ type: "blank" });
329    expect(tokens[2]).toMatchObject({ type: "paragraph" });
330    expect(tokens[3]).toEqual({ type: "blank" });
331    expect(tokens[4]).toEqual({
332      type: "code_block",
333      lang: "js",
334      content: "const x = 1;",
335    });
336    expect(tokens[5]).toEqual({ type: "blank" });
337    expect(tokens[6]).toMatchObject({
338      type: "list_item",
339      ordered: false,
340      index: 0,
341    });
342    expect(tokens[7]).toMatchObject({
343      type: "list_item",
344      ordered: false,
345      index: 0,
346    });
347    expect(tokens[8]).toEqual({ type: "blank" });
348    expect(tokens[9]).toMatchObject({ type: "blockquote" });
349    expect(tokens[10]).toEqual({ type: "blank" });
350    expect(tokens[11]).toEqual({ type: "hr" });
351  });
352});
353
354// ── Streaming tokenizer ─────────────────────────────────────────────
355
356describe("createTokenizer: streaming", () => {
357  test("basic streaming produces same tokens as batch", () => {
358    const input = "# Hello\n\nSome **bold** text.\n";
359    const batchTokens = tokenize(input);
360
361    const tokenizer = createTokenizer();
362    const streamTokens: BlockToken[] = [];
363    streamTokens.push(...tokenizer.write(input));
364    streamTokens.push(...tokenizer.end());
365
366    expect(streamTokens).toEqual(batchTokens);
367  });
368
369  test("character-by-character streaming", () => {
370    const input = "# Title\n\n- item 1\n- item 2\n";
371    const batchTokens = tokenize(input);
372
373    const tokenizer = createTokenizer();
374    const streamTokens: BlockToken[] = [];
375    for (const ch of input) {
376      streamTokens.push(...tokenizer.write(ch));
377    }
378    streamTokens.push(...tokenizer.end());
379
380    expect(streamTokens).toEqual(batchTokens);
381  });
382
383  test("chunks split across line boundaries", () => {
384    const tokenizer = createTokenizer();
385    const tokens: BlockToken[] = [];
386
387    tokens.push(...tokenizer.write("# Hea"));
388    tokens.push(...tokenizer.write("ding\n"));
389    tokens.push(...tokenizer.write("\nSome text\n"));
390    tokens.push(...tokenizer.end());
391
392    expect(tokens[0]).toEqual({
393      type: "heading",
394      depth: 1,
395      spans: [{ type: "text", content: "Heading" }],
396    });
397    expect(tokens[1]).toEqual({ type: "blank" });
398    expect(tokens[2]).toEqual({
399      type: "paragraph",
400      spans: [{ type: "text", content: "Some text" }],
401    });
402  });
403
404  test("emits completed tokens immediately", () => {
405    const tokenizer = createTokenizer();
406
407    // Incomplete line — no token emitted yet
408    const t1 = tokenizer.write("# Hel");
409    expect(t1).toEqual([]);
410
411    // Complete the line — token emitted
412    const t2 = tokenizer.write("lo\n");
413    expect(t2).toEqual([
414      {
415        type: "heading",
416        depth: 1,
417        spans: [{ type: "text", content: "Hello" }],
418      },
419    ]);
420  });
421
422  test("buffers only incomplete line", () => {
423    const tokenizer = createTokenizer();
424
425    const t1 = tokenizer.write("hello\nwor");
426    expect(t1).toEqual([
427      { type: "paragraph", spans: [{ type: "text", content: "hello" }] },
428    ]);
429
430    // "wor" is buffered, flushed on end()
431    const t2 = tokenizer.end();
432    expect(t2).toEqual([
433      { type: "paragraph", spans: [{ type: "text", content: "wor" }] },
434    ]);
435  });
436
437  test("code block spanning multiple writes", () => {
438    const tokenizer = createTokenizer();
439    const tokens: BlockToken[] = [];
440
441    tokens.push(...tokenizer.write("```js\ncon"));
442    tokens.push(...tokenizer.write("st x = 1;\n```\n"));
443    tokens.push(...tokenizer.end());
444
445    expect(tokens).toContainEqual({
446      type: "code_block",
447      lang: "js",
448      content: "const x = 1;",
449    });
450  });
451
452  test("unclosed code block flushed on end()", () => {
453    const tokenizer = createTokenizer();
454    const tokens: BlockToken[] = [];
455
456    tokens.push(...tokenizer.write("```\nsome code"));
457    tokens.push(...tokenizer.end());
458
459    expect(tokens).toContainEqual({
460      type: "code_block",
461      lang: "",
462      content: "some code",
463    });
464  });
465
466  test("accepts Uint8Array chunks", () => {
467    const tokenizer = createTokenizer();
468    const tokens: BlockToken[] = [];
469
470    tokens.push(...tokenizer.write(new TextEncoder().encode("# Hello\n")));
471    tokens.push(...tokenizer.end());
472
473    expect(tokens).toEqual([
474      {
475        type: "heading",
476        depth: 1,
477        spans: [{ type: "text", content: "Hello" }],
478      },
479    ]);
480  });
481
482  test("random chunk sizes match batch", () => {
483    const input =
484      "```javascript\nconst x = 1;\n```\n\n# Heading 2\n\nSome text with `code`.\n> Blockquote here\n\n1. One\n2. Two\n";
485    const batchTokens = tokenize(input);
486
487    const tokenizer = createTokenizer();
488    const streamTokens: BlockToken[] = [];
489
490    let i = 0;
491    while (i < input.length) {
492      const chunkSize = Math.floor(Math.random() * 10) + 1;
493      streamTokens.push(...tokenizer.write(input.slice(i, i + chunkSize)));
494      i += chunkSize;
495    }
496    streamTokens.push(...tokenizer.end());
497
498    expect(streamTokens).toEqual(batchTokens);
499  });
500});
501
502// ── Performance ──────────────────────────────────────────────────────
503
504describe("tokenizer: performance", () => {
505  test("batch tokenize 5000 blocks under 200ms", () => {
506    const largeMd = "# Heading\n\nSome **bold** text here.\n".repeat(5000);
507    const start = performance.now();
508    tokenize(largeMd);
509    const duration = performance.now() - start;
510    console.log(`Token batch parse time: ${duration.toFixed(2)}ms`);
511    expect(duration).toBeLessThan(200);
512  });
513});