@@ -0,0 +1,513 @@
1+import { test, expect, describe } from "bun:test";
2+import { tokenize, createTokenizer, type BlockToken } from "../src/index";
3+
4+// ── Inline span parsing ──────────────────────────────────────────────
5+
6+describe("tokenize: inline spans", () => {
7+ test("plain text", () => {
8+ const tokens = tokenize("Hello world");
9+ expect(tokens).toEqual([
10+ { type: "paragraph", spans: [{ type: "text", content: "Hello world" }] },
11+ ]);
12+ });
13+
14+ test("bold", () => {
15+ const tokens = tokenize("some **bold** text");
16+ expect(tokens).toEqual([
17+ {
18+ type: "paragraph",
19+ spans: [
20+ { type: "text", content: "some " },
21+ { type: "bold", content: "bold" },
22+ { type: "text", content: " text" },
23+ ],
24+ },
25+ ]);
26+ });
27+
28+ test("italic", () => {
29+ const tokens = tokenize("some *italic* text");
30+ expect(tokens).toEqual([
31+ {
32+ type: "paragraph",
33+ spans: [
34+ { type: "text", content: "some " },
35+ { type: "italic", content: "italic" },
36+ { type: "text", content: " text" },
37+ ],
38+ },
39+ ]);
40+ });
41+
42+ test("inline code", () => {
43+ const tokens = tokenize("use `foo()` here");
44+ expect(tokens).toEqual([
45+ {
46+ type: "paragraph",
47+ spans: [
48+ { type: "text", content: "use " },
49+ { type: "code", content: "foo()" },
50+ { type: "text", content: " here" },
51+ ],
52+ },
53+ ]);
54+ });
55+
56+ test("link", () => {
57+ const tokens = tokenize("click [here](https://example.com) now");
58+ expect(tokens).toEqual([
59+ {
60+ type: "paragraph",
61+ spans: [
62+ { type: "text", content: "click " },
63+ { type: "link", text: "here", url: "https://example.com" },
64+ { type: "text", content: " now" },
65+ ],
66+ },
67+ ]);
68+ });
69+
70+ test("mixed inline formatting", () => {
71+ const tokens = tokenize("**bold** and *italic* and `code`");
72+ expect(tokens).toEqual([
73+ {
74+ type: "paragraph",
75+ spans: [
76+ { type: "bold", content: "bold" },
77+ { type: "text", content: " and " },
78+ { type: "italic", content: "italic" },
79+ { type: "text", content: " and " },
80+ { type: "code", content: "code" },
81+ ],
82+ },
83+ ]);
84+ });
85+
86+ test("unclosed bold falls back to text", () => {
87+ const tokens = tokenize("some **unclosed bold");
88+ expect(tokens).toEqual([
89+ {
90+ type: "paragraph",
91+ spans: [{ type: "text", content: "some **unclosed bold" }],
92+ },
93+ ]);
94+ });
95+
96+ test("unclosed italic falls back to text", () => {
97+ const tokens = tokenize("some *unclosed italic");
98+ expect(tokens).toEqual([
99+ {
100+ type: "paragraph",
101+ spans: [{ type: "text", content: "some *unclosed italic" }],
102+ },
103+ ]);
104+ });
105+
106+ test("unclosed inline code falls back to text", () => {
107+ const tokens = tokenize("some `unclosed code");
108+ expect(tokens).toEqual([
109+ {
110+ type: "paragraph",
111+ spans: [{ type: "text", content: "some `unclosed code" }],
112+ },
113+ ]);
114+ });
115+
116+ test("inline code takes priority over bold", () => {
117+ const tokens = tokenize("`**not bold**`");
118+ expect(tokens).toEqual([
119+ {
120+ type: "paragraph",
121+ spans: [{ type: "code", content: "**not bold**" }],
122+ },
123+ ]);
124+ });
125+});
126+
127+// ── Block-level tokens ───────────────────────────────────────────────
128+
129+describe("tokenize: block tokens", () => {
130+ test("heading 1", () => {
131+ const tokens = tokenize("# Hello");
132+ expect(tokens).toEqual([
133+ {
134+ type: "heading",
135+ depth: 1,
136+ spans: [{ type: "text", content: "Hello" }],
137+ },
138+ ]);
139+ });
140+
141+ test("heading 2", () => {
142+ const tokens = tokenize("## Sub");
143+ expect(tokens).toEqual([
144+ {
145+ type: "heading",
146+ depth: 2,
147+ spans: [{ type: "text", content: "Sub" }],
148+ },
149+ ]);
150+ });
151+
152+ test("heading 3", () => {
153+ const tokens = tokenize("### Third");
154+ expect(tokens).toEqual([
155+ {
156+ type: "heading",
157+ depth: 3,
158+ spans: [{ type: "text", content: "Third" }],
159+ },
160+ ]);
161+ });
162+
163+ test("heading with inline formatting", () => {
164+ const tokens = tokenize("# **Bold** heading");
165+ expect(tokens).toEqual([
166+ {
167+ type: "heading",
168+ depth: 1,
169+ spans: [
170+ { type: "bold", content: "Bold" },
171+ { type: "text", content: " heading" },
172+ ],
173+ },
174+ ]);
175+ });
176+
177+ test("fenced code block", () => {
178+ const tokens = tokenize("```js\nconst x = 1;\n```");
179+ expect(tokens).toEqual([
180+ { type: "code_block", lang: "js", content: "const x = 1;" },
181+ ]);
182+ });
183+
184+ test("fenced code block without language", () => {
185+ const tokens = tokenize("```\nhello\n```");
186+ expect(tokens).toEqual([
187+ { type: "code_block", lang: "", content: "hello" },
188+ ]);
189+ });
190+
191+ test("fenced code block with multiple lines", () => {
192+ const tokens = tokenize("```\nline1\nline2\nline3\n```");
193+ expect(tokens).toEqual([
194+ { type: "code_block", lang: "", content: "line1\nline2\nline3" },
195+ ]);
196+ });
197+
198+ test("unclosed code block emits as code_block on end", () => {
199+ const tokens = tokenize("```\nunclosed code");
200+ expect(tokens).toEqual([
201+ { type: "code_block", lang: "", content: "unclosed code" },
202+ ]);
203+ });
204+
205+ test("unordered list item with dash", () => {
206+ const tokens = tokenize("- Item one");
207+ expect(tokens).toEqual([
208+ {
209+ type: "list_item",
210+ ordered: false,
211+ index: 0,
212+ spans: [{ type: "text", content: "Item one" }],
213+ },
214+ ]);
215+ });
216+
217+ test("unordered list item with asterisk", () => {
218+ const tokens = tokenize("* Item two");
219+ expect(tokens).toEqual([
220+ {
221+ type: "list_item",
222+ ordered: false,
223+ index: 0,
224+ spans: [{ type: "text", content: "Item two" }],
225+ },
226+ ]);
227+ });
228+
229+ test("ordered list item", () => {
230+ const tokens = tokenize("1. First");
231+ expect(tokens).toEqual([
232+ {
233+ type: "list_item",
234+ ordered: true,
235+ index: 1,
236+ spans: [{ type: "text", content: "First" }],
237+ },
238+ ]);
239+ });
240+
241+ test("ordered list preserves index", () => {
242+ const tokens = tokenize("3. Third");
243+ expect(tokens).toEqual([
244+ {
245+ type: "list_item",
246+ ordered: true,
247+ index: 3,
248+ spans: [{ type: "text", content: "Third" }],
249+ },
250+ ]);
251+ });
252+
253+ test("blockquote", () => {
254+ const tokens = tokenize("> quoted text");
255+ expect(tokens).toEqual([
256+ {
257+ type: "blockquote",
258+ spans: [{ type: "text", content: "quoted text" }],
259+ },
260+ ]);
261+ });
262+
263+ test("blockquote with inline formatting", () => {
264+ const tokens = tokenize("> **bold** quote");
265+ expect(tokens).toEqual([
266+ {
267+ type: "blockquote",
268+ spans: [
269+ { type: "bold", content: "bold" },
270+ { type: "text", content: " quote" },
271+ ],
272+ },
273+ ]);
274+ });
275+
276+ test("horizontal rule ---", () => {
277+ const tokens = tokenize("---");
278+ expect(tokens).toEqual([{ type: "hr" }]);
279+ });
280+
281+ test("horizontal rule ***", () => {
282+ const tokens = tokenize("***");
283+ expect(tokens).toEqual([{ type: "hr" }]);
Cut short: this commit's diff is over 1000 lines.