xonecas.io
719
Le

lextide / files

lextide / test / lextide.test.ts

20.3 KB
1import { describe, expect, test } from "bun:test";
2
3import {
4	all,
5	common,
6	createLowlight,
7	type ElementContent,
8	type LanguageFn,
9	type Root,
10	type StreamUpdate,
11} from "../src/index";
12
13const toyLanguage: LanguageFn = () => ({
14	name: "toy",
15	relevance: 1,
16	startState: "root",
17	states: {
18		root: [
19			{ match: /foo/, scope: "keyword", relevance: 3 },
20			{ match: /bar/, scope: "title.function", relevance: 2 },
21			{ match: /"[^"]*"/, scope: "string", relevance: 1 },
22		],
23	},
24});
25
26const samples = {
27	bash: '#!/usr/bin/env bash\nif [ -n "$USER" ]; then\n  echo "hi"\nfi\n# done\n',
28	javascript:
29		'function greet(name) {\n  const total = 1 + 2\n  return "hi" + name\n}\n// done\n',
30	markdown:
31		"# Hello\n\n- **bold** and _italic_\n> quote with `code`\n\n[docs](https://example.com) &\n```ts\nconst value = 1\n```\n",
32	python: 'def greet(name):\n    return """hi\nname\n"""\n# done\n',
33	typescript:
34		'interface Person {\n  name: string\n}\nconst value: Person = {name: "Ada"}\n',
35} as const;
36
37describe("lextide", () => {
38	test("builtins_commonAndAll_exportSupportedLanguages", () => {
39		expect(Object.keys(common)).toEqual([
40			"bash",
41			"javascript",
42			"typescript",
43			"python",
44			"markdown",
45		]);
46		expect(Object.keys(all)).toEqual([
47			"bash",
48			"javascript",
49			"typescript",
50			"python",
51			"markdown",
52		]);
53	});
54
55	test("createLowlight_listLanguages_withBuiltins_returnsCanonicalNames", () => {
56		const lowlight = createLowlight(common);
57
58		expect(lowlight.listLanguages()).toEqual([
59			"bash",
60			"javascript",
61			"typescript",
62			"python",
63			"markdown",
64		]);
65	});
66
67	test("highlight_unknownLanguage_throwsLowlightStyleError", () => {
68		const lowlight = createLowlight(common);
69
70		expect(() => lowlight.highlight("unknown", "const value = 1")).toThrow(
71			"Unknown language: `unknown` is not registered",
72		);
73	});
74
75	test("register_and_registerAlias_overloads_resolveCustomGrammar", () => {
76		const lowlight = createLowlight();
77
78		lowlight.register({ toy: toyLanguage });
79		lowlight.registerAlias({ toy: ["toybox", "toy-script"] });
80
81		expect(lowlight.listLanguages()).toEqual(["toy"]);
82		expect(lowlight.registered("toy")).toBe(true);
83		expect(lowlight.registered("toybox")).toBe(true);
84		expect(lowlight.registered("toy-script")).toBe(true);
85		expect(lowlight.registered("missing")).toBe(false);
86
87		const tree = lowlight.highlight("toybox", "foo bar");
88
89		expect(tree.data?.language).toBe("toybox");
90		expect(tree.data?.relevance).toBe(6);
91		expect(simplifyChildren(tree.children)).toEqual([
92			["span", ["hljs-keyword"], ["foo"]],
93			["text", " "],
94			["span", ["hljs-title", "function_"], ["bar"]],
95		]);
96	});
97
98	test("highlight_prefixOptions_serializeScopedClassesLikeLowlight", () => {
99		const lowlight = createLowlight({ toy: toyLanguage });
100
101		expect(firstClassNames(lowlight.highlight("toy", "bar"))).toEqual([
102			"hljs-title",
103			"function_",
104		]);
105		expect(
106			firstClassNames(lowlight.highlight("toy", "bar", { prefix: null })),
107		).toEqual(["title", "function_"]);
108		expect(
109			firstClassNames(lowlight.highlight("toy", "bar", { prefix: "code-" })),
110		).toEqual(["code-title", "function_"]);
111	});
112
113	test("highlightAuto_emptyRegistry_returnsEmptyRoot", () => {
114		const lowlight = createLowlight();
115
116		expect(lowlight.highlightAuto("const value = 1")).toEqual({
117			type: "root",
118			children: [],
119			data: { language: undefined, relevance: 0 },
120		});
121	});
122
123	test("highlightAuto_subsetFiltersCandidates_andPicksBestMatch", () => {
124		const lowlight = createLowlight(common);
125
126		const result = lowlight.highlightAuto(
127			"interface Person {\n  name: string\n}\n",
128			{ subset: ["javascript", "typescript", "unknown"] },
129		);
130
131		expect(result.data?.language).toBe("typescript");
132		expect(result.data?.relevance).toBeGreaterThan(0);
133	});
134
135	test("highlightAuto_noUsefulMatch_returnsEmptyRoot", () => {
136		const lowlight = createLowlight(common);
137		const result = lowlight.highlightAuto("blorf snarg flerm glip");
138
139		expect(result).toEqual({
140			type: "root",
141			children: [],
142			data: { language: undefined, relevance: 0 },
143		});
144	});
145
146	test("builtInAliases_highlightRequestedAlias_andReportAliasInData", () => {
147		const lowlight = createLowlight(common);
148
149		expect(lowlight.registered("js")).toBe(true);
150		expect(lowlight.registered("ts")).toBe(true);
151		expect(lowlight.registered("py")).toBe(true);
152		expect(lowlight.registered("sh")).toBe(true);
153		expect(lowlight.registered("md")).toBe(true);
154
155		expect(lowlight.highlight("js", "const value = 1").data?.language).toBe(
156			"js",
157		);
158		expect(lowlight.highlight("ts", "interface Person {}").data?.language).toBe(
159			"ts",
160		);
161		expect(
162			lowlight.highlight("py", "def greet():\n    return 1\n").data?.language,
163		).toBe("py");
164		expect(lowlight.highlight("sh", 'echo "$USER"\n').data?.language).toBe(
165			"sh",
166		);
167		expect(lowlight.highlight("md", "# Title\n").data?.language).toBe("md");
168	});
169
170	test("highlight_bash_returnsExpectedScopesAndMetadata", () => {
171		const lowlight = createLowlight(common);
172		const tree = lowlight.highlight("bash", samples.bash);
173
174		expect(tree.type).toBe("root");
175		expect(tree.data?.language).toBe("bash");
176		expect(tree.data?.relevance).toBeGreaterThan(0);
177		expect(collectClasses(tree)).toEqual(
178			expect.arrayContaining([
179				"hljs-comment",
180				"hljs-keyword",
181				"hljs-string",
182				"hljs-variable",
183			]),
184		);
185	});
186
187	test("highlight_javascript_returnsExpectedScopesAndMetadata", () => {
188		const lowlight = createLowlight(common);
189		const tree = lowlight.highlight("javascript", samples.javascript);
190
191		expect(tree.type).toBe("root");
192		expect(tree.data?.language).toBe("javascript");
193		expect(tree.data?.relevance).toBeGreaterThan(0);
194		expect(collectClasses(tree)).toEqual(
195			expect.arrayContaining([
196				"hljs-comment",
197				"hljs-keyword",
198				"hljs-number",
199				"hljs-string",
200				"hljs-title",
201			]),
202		);
203	});
204
205	test("highlight_typescript_returnsExpectedScopesAndMetadata", () => {
206		const lowlight = createLowlight(common);
207		const tree = lowlight.highlight("typescript", samples.typescript);
208
209		expect(tree.type).toBe("root");
210		expect(tree.data?.language).toBe("typescript");
211		expect(tree.data?.relevance).toBeGreaterThan(0);
212		expect(collectClasses(tree)).toEqual(
213			expect.arrayContaining([
214				"hljs-keyword",
215				"hljs-title",
216				"hljs-built_in",
217				"hljs-string",
218			]),
219		);
220	});
221
222	test("highlight_python_returnsExpectedScopesAndMetadata", () => {
223		const lowlight = createLowlight(common);
224		const tree = lowlight.highlight("python", samples.python);
225
226		expect(tree.type).toBe("root");
227		expect(tree.data?.language).toBe("python");
228		expect(tree.data?.relevance).toBeGreaterThan(0);
229		expect(collectClasses(tree)).toEqual(
230			expect.arrayContaining([
231				"hljs-comment",
232				"hljs-keyword",
233				"hljs-string",
234				"hljs-title",
235			]),
236		);
237	});
238
239	test("highlight_markdown_returnsExpectedScopesAndMetadata", () => {
240		const lowlight = createLowlight(common);
241		const tree = lowlight.highlight("markdown", samples.markdown);
242
243		expect(tree.type).toBe("root");
244		expect(tree.data?.language).toBe("markdown");
245		expect(tree.data?.relevance).toBeGreaterThan(0);
246		expect(collectClasses(tree)).toEqual(
247			expect.arrayContaining([
248				"hljs-section",
249				"hljs-bullet",
250				"hljs-strong",
251				"hljs-emphasis",
252				"hljs-quote",
253				"hljs-code",
254				"hljs-string",
255				"hljs-link",
256				"hljs-literal",
257			]),
258		);
259		expect(plainText(tree)).toBe(samples.markdown);
260	});
261
262	test("highlight_malformedSource_doesBestEffortWithoutThrowing", () => {
263		const lowlight = createLowlight(common);
264
265		expect(() =>
266			lowlight.highlight("javascript", "const value = `unterminated"),
267		).not.toThrow();
268		expect(() =>
269			lowlight.highlight("python", "def greet(:\n    pass"),
270		).not.toThrow();
271		expect(() =>
272			lowlight.highlight("bash", 'echo "unterminated'),
273		).not.toThrow();
274	});
275
276	test("bash_heredoc_sameLineContinuation_isNotColoredAsString", () => {
277		const lowlight = createLowlight(common);
278		const tree = lowlight.highlight(
279			"bash",
280			'echo "ok" <<-EOF | cat\n\tHello\nEOF\n',
281		);
282
283		expect(textForClass(tree, "hljs-string")).toContain("<<-EOF");
284		expect(textForClass(tree, "hljs-string")).toContain("\n\tHello\nEOF");
285		expect(textForClass(tree, "hljs-string")).not.toContain("| cat");
286	});
287
288	test("bash_chainedCommands_highlightCommonEnvironmentCommands", () => {
289		const lowlight = createLowlight(common);
290		const tree = lowlight.highlight(
291			"bash",
292			"cat file.txt | rg needle && sed -n '1p' | grep needle\n",
293		);
294
295		expect(textForClass(tree, "hljs-built_in")).toContain("cat");
296		expect(textForClass(tree, "hljs-built_in")).toContain("rg");
297		expect(textForClass(tree, "hljs-built_in")).toContain("sed");
298		expect(textForClass(tree, "hljs-built_in")).toContain("grep");
299	});
300
301	test("bash_commandSubstitutionArithmeticAndArrays_areHighlighted", () => {
302		const lowlight = createLowlight(common);
303		const tree = lowlight.highlight(
304			"bash",
305			'items=("a" "b")\ncount=$((1 + $' +
306				"{#items[@]}))\nresult=$(grep needle file | sed -n '1p')\n",
307		);
308
309		expect(collectClasses(tree)).toEqual(
310			expect.arrayContaining([
311				"hljs-built_in",
312				"hljs-number",
313				"hljs-string",
314				"hljs-subst",
315				"hljs-variable",
316			]),
317		);
318		expect(textForClass(tree, "hljs-variable")).toContain("items");
319		expect(textForClass(tree, "hljs-variable")).toContain("count");
320		expect(textForClass(tree, "hljs-variable")).toContain("result");
321		expect(textForClass(tree, "hljs-built_in")).toContain("grep");
322		expect(textForClass(tree, "hljs-built_in")).toContain("sed");
323	});
324
325	test("bash_commandSubstitution_nestedStrings_keepInnerStringScopes", () => {
326		const lowlight = createLowlight(common);
327		const tree = lowlight.highlight(
328			"bash",
329			'result=$(dirname "' + "$" + "{BASH_SOURCE[0]}" + '")\n',
330		);
331
332		expect(collectClasses(tree)).toEqual(
333			expect.arrayContaining([
334				"hljs-built_in",
335				"hljs-string",
336				"hljs-subst",
337				"hljs-variable",
338			]),
339		);
340		expect(textForClass(tree, "hljs-built_in")).toContain("dirname");
341		expect(textForClass(tree, "hljs-string")).toContain(
342			'"' + "$" + "{BASH_SOURCE[0]}" + '"',
343		);
344	});
345
346	test("bash_functionBody_commandsRemainBuiltIns_afterOpeningBrace", () => {
347		const lowlight = createLowlight(common);
348		const tree = lowlight.highlight(
349			"bash",
350			'function greet() { echo "hi"; }\n',
351		);
352
353		expect(textForClass(tree, "hljs-built_in")).toContain("echo");
354	});
355
356	test("bash_quotedHeredocDelimiter_keepsBodyAsString", () => {
357		const lowlight = createLowlight(common);
358		const tree = lowlight.highlight(
359			"bash",
360			"cat <<'EOF'\n$USER and $(whoami)\nEOF\n",
361		);
362
363		expect(textForClass(tree, "hljs-string")).toContain("$USER and $(whoami)");
364		expect(textForClass(tree, "hljs-variable")).not.toContain("USER");
365	});
366
367	test("javascript_jsx_inlineTags_areHighlighted_withoutLosingEmbeddedCode", () => {
368		const lowlight = createLowlight(common);
369		const tree = lowlight.highlight(
370			"javascript",
371			"const Count = ({count}) => <div>{count > 10 ? <BigCount /> : <SmallCount />}</div>",
372		);
373
374		expect(collectClasses(tree)).toEqual(
375			expect.arrayContaining(["hljs-keyword", "hljs-tag", "hljs-name"]),
376		);
377		expect(textForClass(tree, "hljs-name")).toContain("div");
378		expect(textForClass(tree, "hljs-name")).toContain("BigCount");
379		expect(plainText(tree)).toContain("count > 10 ? ");
380	});
381
382	test("javascript_regexLiterals_areRecognized", () => {
383		const lowlight = createLowlight(common);
384		const tree = lowlight.highlight("javascript", "const x = /a+/g");
385
386		expect(collectClasses(tree)).toContain("hljs-regexp");
387		expect(textForClass(tree, "hljs-regexp")).toContain("/a+/g");
388	});
389
390	test("javascript_postfixIncrementDivision_isNotTreatedAsRegexp", () => {
391		const lowlight = createLowlight(common);
392		const tree = lowlight.highlight("javascript", "x++/foo/g\n");
393
394		expect(collectClasses(tree)).not.toContain("hljs-regexp");
395		expect(plainText(tree)).toBe("x++/foo/g\n");
396	});
397
398	test("javascript_propertyAccess_highlightsOptionalAndDottedMembers", () => {
399		const lowlight = createLowlight(common);
400		const tree = lowlight.highlight(
401			"javascript",
402			"const value = foo?.bar ?? baz\nimport.meta.env\n",
403		);
404
405		expect(collectClasses(tree)).toContain("hljs-property");
406		expect(textForClass(tree, "hljs-property")).toContain("bar");
407		expect(textForClass(tree, "hljs-property")).toContain("meta");
408		expect(textForClass(tree, "hljs-property")).toContain("env");
409	});
410
411	test("javascript_extendsClauses_highlightInheritedClassNames", () => {
412		const lowlight = createLowlight(common);
413		const tree = lowlight.highlight("javascript", "class A extends B {}\n");
414
415		expect(collectClasses(tree)).toContain("inherited__");
416		expect(textForClass(tree, "inherited__")).toContain("B");
417	});
418
419	test("typescript_angleBracketTypeAssertion_isNotTreatedAsJsxTag", () => {
420		const lowlight = createLowlight(common);
421		const tree = lowlight.highlight(
422			"typescript",
423			"const items = <string[]>value as string[]",
424		);
425
426		expect(collectClasses(tree)).not.toContain("hljs-tag");
427		expect(collectClasses(tree)).toContain("hljs-built_in");
428	});
429
430	test("typescript_bareAngleBracketAssertion_isNotTreatedAsJsxTag", () => {
431		const lowlight = createLowlight(common);
432		const tree = lowlight.highlight("typescript", "const value = <T>input\n");
433
434		expect(collectClasses(tree)).not.toContain("hljs-tag");
435		expect(collectClasses(tree)).not.toContain("hljs-name");
436		expect(plainText(tree)).toBe("const value = <T>input\n");
437	});
438
439	test("typescript_genericTypeSyntax_isNotTreatedAsJsxTags", () => {
440		const lowlight = createLowlight(common);
441		const tree = lowlight.highlight(
442			"typescript",
443			"type Maybe<T> = T | null\nconst xs: Array<string> = []\ninterface Person<T> { name: string }\n",
444		);
445
446		expect(collectClasses(tree)).not.toContain("hljs-tag");
447		expect(collectClasses(tree)).not.toContain("hljs-name");
448		expect(collectClasses(tree)).toEqual(
449			expect.arrayContaining([
450				"hljs-attr",
451				"hljs-built_in",
452				"hljs-keyword",
453				"hljs-literal",
454				"hljs-title",
455			]),
456		);
457		expect(plainText(tree)).toContain("Maybe<T>");
458		expect(textForClass(tree, "hljs-attr")).toContain("xs");
459		expect(textForClass(tree, "hljs-attr")).toContain("name");
460		expect(textForClass(tree, "hljs-title")).toContain("Array");
461		expect(textForClass(tree, "hljs-built_in")).toContain("string");
462	});
463
464	test("python_singleQuotedStrings_doNotLeakAcrossNewlines", () => {
465		const lowlight = createLowlight(common);
466		const tree = lowlight.highlight(
467			"python",
468			'value = \'unterminated\nprint("ok")\n',
469		);
470
471		expect(textForClass(tree, "hljs-string")).not.toContain("print");
472		expect(collectClasses(tree)).toContain("hljs-built_in");
473	});
474
475	test("python_templateStrings_areRecognized_andKeepInterpolationSeparate", () => {
476		const lowlight = createLowlight(common);
477		const tree = lowlight.highlight("python", 'value = t"hello {name}"');
478
479		expect(collectClasses(tree)).toEqual(
480			expect.arrayContaining(["hljs-string", "hljs-subst"]),
481		);
482	});
483
484	test("python_decorators_areRecognizedAsMeta", () => {
485		const lowlight = createLowlight(common);
486		const tree = lowlight.highlight(
487			"python",
488			"@decorator\ndef greet():\n    return 1\n",
489		);
490
491		expect(collectClasses(tree)).toContain("hljs-meta");
492		expect(textForClass(tree, "hljs-meta")).toContain("@decorator");
493	});
494
495	test("python_matrixMultiply_doesNotTreatAtSignAsDecorator", () => {
496		const lowlight = createLowlight(common);
497		const tree = lowlight.highlight("python", "result = left@right\n");
498
499		expect(collectClasses(tree)).not.toContain("hljs-meta");
500		expect(plainText(tree)).toBe("result = left@right\n");
501	});
502
503	test("unicodeIdentifiers_areSupportedInDeclarations", () => {
504		const lowlight = createLowlight(common);
505
506		expect(
507			textForClass(
508				lowlight.highlight("javascript", "function привет() {}"),
509				"hljs-title",
510			),
511		).toContain("привет");
512		expect(
513			textForClass(
514				lowlight.highlight("typescript", "interface Привет {}"),
515				"hljs-title",
516			),
517		).toContain("Привет");
518		expect(
519			textForClass(
520				lowlight.highlight("python", "def привет():\n    return 1\n"),
521				"hljs-title",
522			),
523		).toContain("привет");
524	});
525
526	test("stream_end_matchesSync_forManySingleSplitBoundaries", () => {
527		const lowlight = createLowlight(common);
528
529		for (const [language, source] of Object.entries(samples)) {
530			const expected = lowlight.highlight(language, source);
531
532			for (let index = 0; index <= source.length; index++) {
533				const stream = lowlight.stream(language);
534				const rendered: ElementContent[] = [];
535
536				if (index > 0) {
537					applyUpdate(rendered, stream.write(source.slice(0, index)));
538				}
539
540				applyUpdate(rendered, stream.end(source.slice(index)));
541
542				expect(rendered).toEqual(expected.children);
543				expect(stream.snapshot()).toEqual(expected);
544			}
545		}
546	});
547
548	test("stream_lineByLineInput_matchesSyncOutput", () => {
549		const lowlight = createLowlight(common);
550		const source = samples.typescript;
551		const expected = lowlight.highlight("typescript", source);
552		const stream = lowlight.stream("typescript");
553		const rendered: ElementContent[] = [];
554
555		for (const line of source.split(/(?<=\n)/)) {
556			if (line.length > 0) {
557				applyUpdate(rendered, stream.write(line));
558			}
559		}
560
561		applyUpdate(rendered, stream.end());
562
563		expect(rendered).toEqual(expected.children);
564		expect(stream.snapshot()).toEqual(expected);
565	});
566
567	test("stream_oneCharacterAtATime_matchesSyncOutput", () => {
568		const lowlight = createLowlight(common);
569		const source = samples.python;
570		const expected = lowlight.highlight("python", source);
571		const stream = lowlight.stream("python");
572		const rendered: ElementContent[] = [];
573
574		for (const character of source) {
575			applyUpdate(rendered, stream.write(character));
576		}
577
578		applyUpdate(rendered, stream.end());
579
580		expect(rendered).toEqual(expected.children);
581		expect(stream.snapshot()).toEqual(expected);
582	});
583
584	test("stream_allowRecalls_replacesUnstableTail_andSnapshotMatchesRenderedState", () => {
585		const lowlight = createLowlight(common);
586		const stream = lowlight.stream("javascript", { allowRecalls: true });
587		const rendered: ElementContent[] = [];
588
589		const first = stream.write("const value = `hel");
590		applyUpdate(rendered, first);
591
592		expect(first.recall).toBe(0);
593		expect(first.stable).toEqual([]);
594		expect(first.unstable.length).toBeGreaterThan(0);
595		expect(stream.snapshot().children).toEqual(rendered);
596
597		const second = stream.write("lo`;\n");
598		applyUpdate(rendered, second);
599
600		expect(second.recall).toBe(first.unstable.length);
601		expect(stream.snapshot().children).toEqual(rendered);
602
603		const third = stream.end("// done");
604		applyUpdate(rendered, third);
605
606		const expected = lowlight.highlight(
607			"javascript",
608			"const value = `hello`;\n// done",
609		);
610
611		expect(rendered).toEqual(expected.children);
612		expect(stream.snapshot()).toEqual(expected);
613	});
614});
615
616function applyUpdate(rendered: ElementContent[], update: StreamUpdate): void {
617	if (update.recall > 0) {
618		rendered.splice(rendered.length - update.recall, update.recall);
619	}
620
621	rendered.push(...update.stable, ...update.unstable);
622}
623
624function collectClasses(root: Root): string[] {
625	const classes: string[] = [];
626
627	visit(root.children, (node) => {
628		if (node.type === "element") {
629			classes.push(...node.properties.className);
630		}
631	});
632
633	return classes;
634}
635
636function firstClassNames(root: Root): string[] {
637	const first = root.children[0];
638
639	if (!first || first.type !== "element") {
640		throw new Error("expected first child element");
641	}
642
643	return first.properties.className;
644}
645
646function plainText(root: Root): string {
647	return root.children.map((child) => nodeText(child)).join("");
648}
649
650function textForClass(root: Root, className: string): string {
651	const values: string[] = [];
652
653	visit(root.children, (node) => {
654		if (
655			node.type === "element" &&
656			node.properties.className.includes(className)
657		) {
658			values.push(node.children.map((child) => nodeText(child)).join(""));
659		}
660	});
661
662	return values.join("");
663}
664
665function simplifyChildren(children: readonly ElementContent[]): unknown[] {
666	return children.map((child) => simplifyNode(child));
667}
668
669function nodeText(node: ElementContent): string {
670	if (node.type === "text") {
671		return node.value;
672	}
673
674	return node.children.map((child) => nodeText(child)).join("");
675}
676
677function simplifyNode(node: ElementContent): unknown[] {
678	if (node.type === "text") {
679		return ["text", node.value];
680	}
681
682	return [
683		"span",
684		node.properties.className,
685		node.children.map((child) =>
686			child.type === "text" ? child.value : simplifyNode(child),
687		),
688	];
689}
690
691function visit(
692	children: readonly ElementContent[],
693	visitor: (node: ElementContent) => void,
694): void {
695	for (const child of children) {
696		visitor(child);
697
698		if (child.type === "element") {
699			visit(child.children, visitor);
700		}
701	}
702}