xonecas.io
719
Le

lextide / commit e7e9f4e

Add markdown builtin and tighten language parity

xonecas2026-04-11 e7e9f4eb58f02dac9614aeebf5d474eeb8d02a23 parent 26f80b2

11 files changed

This diff is too big to show in full. The first 1000 lines are below, and every file is listed with its counts.

AGENTS.md+3−2
@@ -22,8 +22,8 @@ Before doing anything, read:
2222 - Repo has Bun/TypeScript/Biome/Lefthook scaffold
2323 - Git repo is initialized
2424 - Repo now has working v0 implementation for sync highlighting, auto-detect, registry/aliases, and streaming
25-- Built-in languages currently: bash, javascript, typescript, python
26-- Tests cover public lowlight-style contract plus streaming parity
25+- Built-in languages currently: bash, javascript, typescript, python, markdown
26+- Tests cover public lowlight-style contract, builtin parity cases, and streaming parity
2727 - `spec.md` is still source of truth for future expansion and compatibility gaps
2828
2929 ## Non-negotiables
@@ -75,6 +75,7 @@ Start small:
7575 - javascript
7676 - typescript
7777 - python
78+- markdown
7879
7980 `all` and `common` can be same at first.
8081
README.md+3−1
@@ -17,6 +17,7 @@ Current built-in languages:
1717 - javascript
1818 - typescript
1919 - python
20+- markdown
2021
2122 `common` and `all` same for now.
2223
@@ -96,7 +97,7 @@ import { common, createLowlight } from "lextide";
9697 const lowlight = createLowlight(common);
9798
9899 console.log(lowlight.listLanguages());
99-// ["bash", "javascript", "typescript", "python"]
100+// ["bash", "javascript", "typescript", "python", "markdown"]
100101 ```
101102
102103 Useful built-in aliases:
@@ -104,6 +105,7 @@ Useful built-in aliases:
104105 - javascript: `js`, `jsx`, `mjs`, `cjs`
105106 - typescript: `ts`, `tsx`, `mts`, `cts`
106107 - python: `py`
108+- markdown: `md`, `mkdown`, `mkd`
107109 - bash: `sh`, `shell`, `zsh`, `bash`
108110
109111 ## `highlight()`
TODO.md+7−5
@@ -1,5 +1,8 @@
11 # TODO
22
3+Keep this file minimal.
4+Use it to track progress with the user.
5+
36 - [x] scaffold package, with git, typescript, and bun for tests and package management.
47 - [x] Add minimal bootstrap files.
58 - [x] Install biomejs for formatting and linting. Ensure organizeImports is on and our complexity config matches:
@@ -17,11 +20,10 @@
1720 ```
1821
1922 - [x] Use left-hook to install pre-commit hook that checks test, biome format, biome lint and typescript types.
20-- [x] Update AGENTS.md with the new scripts and process. Correct anything that is outdated or not specified.
21-- [x] Update TODO.md with your progress
2223 - [x] Implement tests to match spec and lowlight user contract
2324 - [x] Implement the spec.
2425 - [x] Review implementationg, fix bugs, ensure complete implementation.
25-- [x] Update AGENTS.md with the new scripts and process. Correct anything that is outdated or not specified.
26-- [x] Update TODO.md with your progress
27-- [ ] complete language parity for builtin languages
26+- [x] Add Markdown support to languages
27+- [x] complete language parity for builtin languages
28+- [x] fix nested state parsing inside child matches (`parseChildren`) so bash command substitutions like `$(dirname "${BASH_SOURCE[0]}")` preserve inner string scopes instead of flattening quotes to plain text
29+- [x] Update AGENTS.md and TODO.md
src/builtins.ts+3−0
@@ -1,5 +1,6 @@
11 import { bash } from "./languages/bash";
22 import { javascript } from "./languages/javascript";
3+import { markdown } from "./languages/markdown";
34 import { python } from "./languages/python";
45 import { typescript } from "./languages/typescript";
56 import type { LanguageFn } from "./types";
@@ -9,6 +10,7 @@ export const common: Readonly<Record<string, LanguageFn>> = {
910 javascript,
1011 typescript,
1112 python,
13+ markdown,
1214 };
1315
1416 export const all: Readonly<Record<string, LanguageFn>> = {
@@ -16,4 +18,5 @@ export const all: Readonly<Record<string, LanguageFn>> = {
1618 javascript,
1719 typescript,
1820 python,
21+ markdown,
1922 };
src/core/engine.ts+73−8
@@ -18,6 +18,11 @@ interface ResolvedRule {
1818 scope?: string | undefined;
1919 }
2020
21+interface ChildState {
22+ defaultScope?: string | undefined;
23+ rules: readonly InternalRule[];
24+}
25+
2126 export interface Segment {
2227 text: string;
2328 scopes: string[];
@@ -141,7 +146,15 @@ function consumeResolvedSegment(
141146 resolved.scope,
142147 );
143148
144- emitResolvedSegments(segments, text, scopes, position, stack, resolved);
149+ emitResolvedSegments(
150+ states,
151+ segments,
152+ text,
153+ scopes,
154+ position,
155+ stack,
156+ resolved,
157+ );
145158 applyStateTransition(stack, resolved);
146159 updateTailState(segments, stack);
147160
@@ -163,6 +176,7 @@ function currentState(
163176 }
164177
165178 function emitResolvedSegments(
179+ states: ReturnType<typeof getInternalStates>,
166180 segments: Segment[],
167181 text: string,
168182 scopes: readonly string[],
@@ -172,6 +186,7 @@ function emitResolvedSegments(
172186 ): void {
173187 if (resolved.children && resolved.children.length > 0) {
174188 const childSegments = parseChildren(
189+ states,
175190 text,
176191 resolved.children,
177192 scopes,
@@ -213,6 +228,26 @@ function applyStateTransition(stack: string[], resolved: ResolvedRule): void {
213228 }
214229 }
215230
231+function applyChildStateTransition(
232+ states: ReturnType<typeof getInternalStates>,
233+ stack: ChildState[],
234+ resolved: ResolvedRule,
235+): void {
236+ if (resolved.push) {
237+ const state = states[resolved.push];
238+
239+ if (!state) {
240+ throw new Error(`Unknown grammar state: ${resolved.push}`);
241+ }
242+
243+ stack.push(state);
244+ }
245+
246+ if (resolved.pop && stack.length > 1) {
247+ stack.pop();
248+ }
249+}
250+
216251 function updateTailState(segments: Segment[], stack: readonly string[]): void {
217252 const tail = segments[segments.length - 1];
218253
@@ -242,6 +277,30 @@ function activeScopes(
242277 return scopes;
243278 }
244279
280+function activeChildScopes(
281+ baseScopes: readonly string[],
282+ stack: readonly ChildState[],
283+ pushScope?: string | null,
284+ transientScope?: string,
285+): string[] {
286+ const scopes = [
287+ ...baseScopes,
288+ ...stack
289+ .map((state) => state.defaultScope)
290+ .filter((scope): scope is string => typeof scope === "string"),
291+ ];
292+
293+ if (typeof pushScope === "string") {
294+ scopes.push(pushScope);
295+ }
296+
297+ if (typeof transientScope === "string") {
298+ scopes.push(transientScope);
299+ }
300+
301+ return scopes;
302+}
303+
245304 function codePointLength(input: string, position: number): number {
246305 const codePoint = input.codePointAt(position);
247306
@@ -295,6 +354,7 @@ function matchStateRules(
295354 }
296355
297356 function parseChildren(
357+ states: ReturnType<typeof getInternalStates>,
298358 input: string,
299359 rules: readonly InternalRule[],
300360 baseScopes: readonly string[],
@@ -302,10 +362,12 @@ function parseChildren(
302362 stateAfter: readonly string[],
303363 ): Segment[] {
304364 const segments: Segment[] = [];
365+ const childStack: ChildState[] = [{ rules }];
305366 let position = 0;
306367
307368 while (position < input.length) {
308- const resolved = matchStateRules(rules, input, position);
369+ const current = childStack[childStack.length - 1];
370+ const resolved = matchStateRules(current.rules, input, position);
309371
310372 if (!resolved) {
311373 const length = codePointLength(input, position);
@@ -314,7 +376,7 @@ function parseChildren(
314376 pushSegment(
315377 segments,
316378 input.slice(position, end),
317- [...baseScopes],
379+ activeChildScopes(baseScopes, childStack),
318380 offset + position,
319381 offset + end,
320382 stateAfter,
@@ -326,14 +388,16 @@ function parseChildren(
326388
327389 const end = position + resolved.length;
328390 const text = input.slice(position, end);
329- const scopes = [...baseScopes];
330-
331- if (resolved.scope) {
332- scopes.push(resolved.scope);
333- }
391+ const scopes = activeChildScopes(
392+ baseScopes,
393+ childStack,
394+ resolved.pushScope,
395+ resolved.scope,
396+ );
334397
335398 if (resolved.children && resolved.children.length > 0) {
336399 const childSegments = parseChildren(
400+ states,
337401 text,
338402 resolved.children,
339403 scopes,
@@ -362,6 +426,7 @@ function parseChildren(
362426 );
363427 }
364428
429+ applyChildStateTransition(states, childStack, resolved);
365430 position = end;
366431 }
367432
src/languages/bash.ts+1−1
@@ -636,7 +636,7 @@ function isCommandStart(input: string, position: number): boolean {
636636 return true;
637637 }
638638
639- if (/[;&|()]/.test(input[cursor])) {
639+ if (/[;&|(){}]/.test(input[cursor])) {
640640 return true;
641641 }
642642
src/languages/ecmascript.ts+397−3
@@ -8,7 +8,9 @@ import {
88 createIdentifierMatcher,
99 createNumberRule,
1010 ECMASCRIPT_IDENTIFIER,
11+ escapeForExpression,
1112 scanIdentifier,
13+ startsWithWord,
1214 } from "./helpers";
1315
1416 interface EcmaScriptDeclaration {
@@ -25,6 +27,7 @@ interface EcmaScriptGrammarConfig {
2527 keywords: ReadonlySet<string>;
2628 literals: ReadonlySet<string>;
2729 name: string;
30+ types?: boolean;
2831 }
2932
3033 const JSX_TAG_CHILDREN: InternalRule[] = [
@@ -35,6 +38,29 @@ const JSX_TAG_CHILDREN: InternalRule[] = [
3538 { match: jsxBraceMatcher, scope: "subst" },
3639 ];
3740
41+const JSX_PREFIX_CHARACTERS = new Set([
42+ "!",
43+ "%",
44+ "&",
45+ "(",
46+ "*",
47+ "+",
48+ ",",
49+ ":",
50+ ";",
51+ "=",
52+ ">",
53+ "?",
54+ "[",
55+ "^",
56+ "{",
57+ "|",
58+ "~",
59+ "-",
60+]);
61+
62+const REGEXP_PREFIX_CHARACTERS = new Set(["/", "<", ...JSX_PREFIX_CHARACTERS]);
63+
3864 export function createEcmaScriptGrammar(
3965 config: EcmaScriptGrammarConfig,
4066 ): GrammarDefinition {
@@ -119,8 +145,9 @@ function createEcmaScriptRules(
119145 pushScope: "string",
120146 relevance: 1,
121147 },
148+ { match: regexpLiteralMatcher, scope: "regexp", relevance: 1 },
122149 {
123- match: jsxTagMatcher,
150+ match: createJsxTagMatcher(config.types === true, config.builtIns),
124151 scope: "tag",
125152 children: JSX_TAG_CHILDREN,
126153 relevance: 1,
@@ -143,11 +170,23 @@ function createEcmaScriptRules(
143170 );
144171 }
145172
173+ rules.push({ match: inheritanceClauseMatcher, relevance: 1 });
174+
175+ if (config.types) {
176+ rules.push({ match: typeAttributeMatcher, scope: "attr", relevance: 0 });
177+ rules.push({
178+ match: typeReferenceMatcher,
179+ scope: "title.class",
180+ relevance: 0,
181+ });
182+ }
183+
146184 if (includeSubstBraces) {
147185 rules.push({ match: /\{/, push: "templateSubst", relevance: 0 });
148186 rules.push({ match: /\}/, pop: true, relevance: 0 });
149187 }
150188
189+ rules.push({ match: propertyMatcher, scope: "property", relevance: 0 });
151190 rules.push(createNumberRule());
152191 rules.push({
153192 match: createIdentifierMatcher({
@@ -193,7 +232,24 @@ function jsxBraceMatcher(input: string, position: number): number | null {
193232 return end === null ? null : end - position;
194233 }
195234
196-function jsxTagMatcher(input: string, position: number): number | null {
235+function createJsxTagMatcher(
236+ types: boolean,
237+ typeBuiltIns?: ReadonlySet<string>,
238+): (input: string, position: number) => number | null {
239+ return (input, position) =>
240+ jsxTagMatcher(input, position, types, typeBuiltIns);
241+}
242+
243+function jsxTagMatcher(
244+ input: string,
245+ position: number,
246+ types = false,
247+ typeBuiltIns?: ReadonlySet<string>,
248+): number | null {
249+ if (!isLikelyJsxStart(input, position)) {
250+ return null;
251+ }
252+
197253 const fragmentLength = jsxFragmentLength(input, position);
198254
199255 if (fragmentLength !== null) {
@@ -208,7 +264,24 @@ function jsxTagMatcher(input: string, position: number): number | null {
208264
209265 const end = skipJsxTagBody(input, prefix.nameEnd);
210266
211- return end === null ? null : end - position;
267+ if (end === null) {
268+ return null;
269+ }
270+
271+ if (
272+ types &&
273+ looksLikeTypeAssertionTag(
274+ input,
275+ position,
276+ prefix.nameEnd,
277+ end,
278+ typeBuiltIns,
279+ )
280+ ) {
281+ return null;
282+ }
283+
284+ return end - position;
212285 }
213286
214287 function jsxFragmentLength(input: string, position: number): number | null {
@@ -263,6 +336,265 @@ function jsxTagPrefix(
263336 return { nameEnd };
264337 }
265338
339+function isLikelyJsxStart(input: string, position: number): boolean {
340+ if (input[position] !== "<") {
341+ return false;
342+ }
343+
344+ const previous = previousSignificantIndex(input, position);
345+
346+ if (previous < 0) {
347+ return true;
348+ }
349+
350+ if (/^[\])}`'"._$\p{ID_Continue}]$/u.test(input[previous])) {
351+ const keyword = previousAsciiWord(input, position);
352+ return (
353+ keyword !== null && /^(?:case|else|return|throw|yield)$/.test(keyword)
354+ );
355+ }
356+
357+ return JSX_PREFIX_CHARACTERS.has(input[previous]);
358+}
359+
360+function previousAsciiWord(input: string, position: number): string | null {
361+ let end = position;
362+
363+ while (end > 0 && /\s/.test(input[end - 1])) {
364+ end -= 1;
365+ }
366+
367+ let start = end;
368+
369+ while (start > 0 && /[A-Za-z]/.test(input[start - 1])) {
370+ start -= 1;
371+ }
372+
373+ return start === end ? null : input.slice(start, end);
374+}
375+
376+function previousSignificantIndex(input: string, position: number): number {
377+ let cursor = position - 1;
378+
379+ while (cursor >= 0 && /\s/.test(input[cursor])) {
380+ cursor -= 1;
381+ }
382+
383+ return cursor;
384+}
385+
386+function looksLikeTypeAssertionTag(
387+ input: string,
388+ position: number,
389+ nameEnd: number,
390+ end: number,
391+ typeBuiltIns?: ReadonlySet<string>,
392+): boolean {
393+ if (input[position + 1] === "/") {
394+ return false;
395+ }
396+
397+ const suffix = input.slice(nameEnd, end - 1).trim();
398+
399+ if (suffix.length > 0) {
400+ return false;
401+ }
402+
403+ const name = input.slice(position + 1, nameEnd);
404+
405+ if (!looksLikeTypeAssertionName(name, typeBuiltIns)) {
406+ return false;
407+ }
408+
409+ return !hasMatchingJsxClosingTagAhead(input, end, name);
410+}
411+
412+function looksLikeTypeAssertionName(
413+ value: string,
414+ typeBuiltIns?: ReadonlySet<string>,
415+): boolean {
416+ return (
417+ /^[A-Z]$/.test(value) ||
418+ looksLikeTypeReference(value) ||
419+ typeBuiltIns?.has(value) === true
420+ );
421+}
422+
423+function hasMatchingJsxClosingTagAhead(
424+ input: string,
425+ position: number,
426+ name: string,
427+): boolean {
428+ const expression = new RegExp(`</${escapeForExpression(name)}\\s*>`);
429+
430+ return expression.test(input.slice(position));
431+}
432+
433+function inheritanceClauseMatcher(
434+ input: string,
435+ position: number,
436+): { children: InternalRule[]; length: number } | null {
437+ const keyword = startsWithWord(input, position, "extends")
438+ ? "extends"
439+ : startsWithWord(input, position, "implements")
440+ ? "implements"
441+ : null;
442+
443+ if (keyword === null) {
444+ return null;
445+ }
446+
447+ const titleStart = skipWhitespace(input, position + keyword.length);
448+ const titleEnd = scanIdentifier(input, titleStart, ECMASCRIPT_IDENTIFIER);
449+
450+ if (titleEnd === null) {
451+ return null;
452+ }
453+
454+ return {
455+ children: [
456+ { match: new RegExp(keyword), scope: "keyword" },
457+ {
458+ match: ECMASCRIPT_IDENTIFIER.pattern,
459+ scope: "title.class.inherited",
460+ },
461+ ],
462+ length: titleEnd - position,
463+ };
464+}
465+
466+function propertyMatcher(input: string, position: number): number | null {
467+ const end = scanIdentifier(input, position, ECMASCRIPT_IDENTIFIER);
468+
469+ if (end === null) {
470+ return null;
471+ }
472+
473+ const previous = previousSignificantIndex(input, position);
474+
475+ if (previous < 0 || input[previous] !== ".") {
476+ return null;
477+ }
478+
479+ return end - position;
480+}
481+
482+function typeAttributeMatcher(input: string, position: number): number | null {
483+ const end = scanIdentifier(input, position, ECMASCRIPT_IDENTIFIER);
484+
485+ if (end === null || !hasTypeAnnotationSuffix(input, end)) {
486+ return null;
487+ }
488+
489+ const previous = previousSignificantIndex(input, position);
490+ const keyword = previousAsciiWord(input, position);
491+
492+ if (
493+ (previous >= 0 && ["{", ",", ";"].includes(input[previous])) ||
494+ (keyword !== null &&
495+ /^(?:const|declare|let|private|protected|public|readonly|var)$/.test(
496+ keyword,
497+ ))
498+ ) {
499+ return end - position;
500+ }
501+
502+ return null;
503+}
504+
505+function typeReferenceMatcher(input: string, position: number): number | null {
506+ const end = scanIdentifier(input, position, ECMASCRIPT_IDENTIFIER);
507+
508+ if (end === null || !looksLikeTypeReference(input.slice(position, end))) {
509+ return null;
510+ }
511+
512+ const previous = previousSignificantIndex(input, position);
513+ const keyword = previousAsciiWord(input, position);
514+
515+ if (
516+ (previous >= 0 &&
517+ ["&", "(", ",", ":", "<", "=", "[", "|"].includes(input[previous])) ||
518+ (keyword !== null && /^(?:as|extends|implements|infer|is)$/.test(keyword))
519+ ) {
520+ return end - position;
521+ }
522+
523+ return null;
524+}
525+
526+function hasTypeAnnotationSuffix(input: string, position: number): boolean {
527+ const probe = skipWhitespace(input, position);
528+
529+ if (input[probe] === ":") {
530+ return true;
531+ }
532+
533+ return input[probe] === "?" && input[probe + 1] === ":";
534+}
535+
536+function looksLikeTypeReference(value: string): boolean {
537+ return value.length > 1 && /^[A-Z]/.test(value);
538+}
539+
540+function regexpLiteralMatcher(input: string, position: number): number | null {
541+ if (
542+ input[position] !== "/" ||
543+ input.startsWith("//", position) ||
544+ input.startsWith("/*", position) ||
545+ !regexpCanStart(input, position)
546+ ) {
547+ return null;
548+ }
549+
550+ const end = skipRegExpLiteral(input, position);
551+
552+ return end === null ? null : end - position;
553+}
554+
555+function regexpCanStart(input: string, position: number): boolean {
556+ const previous = previousSignificantIndex(input, position);
557+
558+ if (previous < 0) {
559+ return true;
560+ }
561+
562+ if (isPostfixUpdateOperator(input, previous)) {
563+ return false;
564+ }
565+
566+ if (REGEXP_PREFIX_CHARACTERS.has(input[previous])) {
567+ return true;
568+ }
569+
570+ const keyword = previousAsciiWord(input, position);
571+
572+ return (
573+ keyword !== null &&
574+ /^(?:await|case|delete|in|instanceof|new|return|throw|typeof|void|yield)$/.test(
575+ keyword,
576+ )
577+ );
578+}
579+
580+function isPostfixUpdateOperator(input: string, position: number): boolean {
581+ const character = input[position];
582+
583+ if (character !== "+" && character !== "-") {
584+ return false;
585+ }
586+
587+ let count = 0;
588+ let cursor = position;
589+
590+ while (cursor >= 0 && input[cursor] === character) {
591+ count += 1;
592+ cursor -= 1;
593+ }
594+
595+ return count === 2;
596+}
597+
266598 function scanJsxName(input: string, position: number): number | null {
267599 let end = scanIdentifier(input, position, ECMASCRIPT_IDENTIFIER);
268600
@@ -368,6 +700,68 @@ function skipJsxTagBody(input: string, position: number): number | null {
368700 return null;
369701 }
370702
703+function skipRegExpLiteral(input: string, position: number): number | null {
704+ let cursor = position + 1;
705+ let inCharacterClass = false;
706+
707+ while (cursor < input.length) {
708+ const advanced = advanceRegExpCursor(input, cursor, inCharacterClass);
709+
710+ if (advanced === null) {
711+ return null;
712+ }
713+
714+ cursor = advanced.cursor;
715+ inCharacterClass = advanced.inCharacterClass;
716+
717+ if (advanced.closed) {
718+ return skipRegExpFlags(input, cursor);
719+ }
720+ }
721+
722+ return null;
723+}
724+
725+function advanceRegExpCursor(
726+ input: string,
727+ cursor: number,
728+ inCharacterClass: boolean,
729+): { closed?: boolean; cursor: number; inCharacterClass: boolean } | null {
730+ const character = input[cursor];
731+
732+ if (character === "\\") {
733+ return { cursor: cursor + 2, inCharacterClass };
734+ }
735+
736+ if (character === "\n") {
737+ return null;
738+ }
739+
740+ if (character === "[" && !inCharacterClass) {
741+ return { cursor: cursor + 1, inCharacterClass: true };
742+ }
743+
744+ if (character === "]" && inCharacterClass) {
745+ return { cursor: cursor + 1, inCharacterClass: false };
746+ }
747+
748+ if (character === "/" && !inCharacterClass) {
749+ return { closed: true, cursor: cursor + 1, inCharacterClass };
750+ }
751+
752+ return { cursor: cursor + 1, inCharacterClass };
753+}
754+
755+function skipRegExpFlags(input: string, position: number): number {
756+ let cursor = position;
757+
758+ while (/[A-Za-z]/.test(input[cursor] ?? "")) {
759+ cursor += 1;
760+ }
761+
762+ return cursor;
763+}
764+
371765 function skipQuotedString(
372766 input: string,
373767 position: number,
src/languages/markdown.ts+376−0
@@ -0,0 +1,376 @@
1+import type { InternalRule } from "../core/grammar";
2+import { defineGrammar } from "../core/grammar";
3+import type { LanguageFn } from "../types";
4+
5+const MARKDOWN_ENTITY = /&(?:[A-Za-z0-9]+|#[0-9]{1,7}|#[Xx][0-9a-fA-F]{1,6});/;
6+
7+const MARKDOWN_INLINE_RULES: InternalRule[] = [
8+ { match: markdownInlineCodeMatcher, scope: "code", relevance: 1 },
9+ { match: markdownLinkMatcher, relevance: 2 },
10+ { match: markdownStrongMatcher, scope: "strong", relevance: 1 },
11+ { match: markdownEmphasisMatcher, scope: "emphasis", relevance: 0 },
12+ { match: MARKDOWN_ENTITY, scope: "literal", relevance: 0 },
13+];
14+
15+export const markdown: LanguageFn = () =>
16+ defineGrammar({
17+ aliases: ["md", "mkdown", "mkd"],
18+ name: "markdown",
19+ relevance: 0,
20+ startState: "root",
21+ states: {
22+ backtickFence: {
23+ defaultScope: "code",
24+ rules: [{ match: markdownBacktickFenceEndMatcher, pop: true }],
25+ },
26+ root: {
27+ rules: [
28+ { match: markdownBacktickFenceStartMatcher, relevance: 3 },
29+ { match: markdownTildeFenceStartMatcher, relevance: 3 },
30+ { match: markdownHeadingMatcher, relevance: 2 },
31+ { match: markdownBlockquoteMatcher, relevance: 1 },
32+ { match: markdownBulletMatcher, scope: "bullet", relevance: 1 },
33+ ...MARKDOWN_INLINE_RULES,
34+ ],
35+ },
36+ tildeFence: {
37+ defaultScope: "code",
38+ rules: [{ match: markdownTildeFenceEndMatcher, pop: true }],
39+ },
40+ },
41+ });
42+
43+function createScopedRanges(
44+ ranges: readonly { length: number; scope: string; start: number }[],
45+): InternalRule[] {
46+ return ranges
47+ .filter((range) => range.length > 0)
48+ .map((range) => ({
49+ match: (_input: string, position: number) =>
50+ position === range.start ? range.length : null,
51+ scope: range.scope,
52+ }));
53+}
54+
55+function findStyledSpanEnd(
56+ input: string,
57+ position: number,
58+ delimiter: string,
59+): number | null {
60+ const lineEnd = findLineEnd(input, position);
61+ const contentStart = position + delimiter.length;
62+
63+ if (contentStart >= lineEnd || /\s/.test(input[contentStart] ?? "")) {
64+ return null;
65+ }
66+
67+ let cursor = input.indexOf(delimiter, contentStart);
68+
69+ while (cursor !== -1 && cursor < lineEnd) {
70+ if (!/\s/.test(input[cursor - 1] ?? "")) {
71+ return cursor + delimiter.length;
72+ }
73+
74+ cursor = input.indexOf(delimiter, cursor + delimiter.length);
75+ }
76+
77+ return null;
78+}
79+
80+function findLineEnd(input: string, position: number): number {
81+ const lineEnd = input.indexOf("\n", position);
82+
83+ return lineEnd === -1 ? input.length : lineEnd;
84+}
85+
86+function findLineEndWithNewline(input: string, position: number): number {
87+ const lineEnd = findLineEnd(input, position);
88+
89+ return lineEnd < input.length ? lineEnd + 1 : lineEnd;
90+}
91+
92+function isFenceLine(
93+ input: string,
94+ position: number,
95+ marker: "`" | "~",
96+): boolean {
97+ if (lineOffset(input, position) === null || input[position] !== marker) {
98+ return false;
99+ }
100+
101+ let cursor = position;
102+
103+ while (input[cursor] === marker) {
104+ cursor += 1;
105+ }
106+
107+ if (cursor - position < 3) {
108+ return false;
109+ }
110+
111+ const lineEnd = findLineEnd(input, position);
112+
113+ for (; cursor < lineEnd; cursor += 1) {
114+ if (input[cursor] !== " " && input[cursor] !== "\t") {
115+ return false;
116+ }
117+ }
118+
119+ return true;
120+}
121+
122+function lineOffset(input: string, position: number): number | null {
123+ let cursor = position;
124+ let offset = 0;
125+
126+ while (cursor > 0 && input[cursor - 1] !== "\n") {
127+ cursor -= 1;
128+ offset += 1;
129+
130+ if (input[cursor] !== " " || offset > 3) {
131+ return null;
132+ }
133+ }
134+
135+ return offset;
136+}
137+
138+function markdownBacktickFenceEndMatcher(
139+ input: string,
140+ position: number,
141+): number | null {
142+ return isFenceLine(input, position, "`")
143+ ? findLineEndWithNewline(input, position) - position
144+ : null;
145+}
146+
147+function markdownBacktickFenceStartMatcher(
148+ input: string,
149+ position: number,
150+): { length: number; push: string; pushScope: string } | null {
151+ return createFenceStartMatch(input, position, "`", "backtickFence");
152+}
153+
154+function markdownBlockquoteMatcher(
155+ input: string,
156+ position: number,
157+): { children: InternalRule[]; length: number; scope: string } | null {
158+ const offset = lineOffset(input, position);
159+
160+ if (offset === null || input[position] !== ">") {
161+ return null;
162+ }
163+
164+ const lineEnd = findLineEnd(input, position);
165+
166+ return {
167+ children: MARKDOWN_INLINE_RULES,
168+ length: lineEnd - position,
169+ scope: "quote",
170+ };
171+}
172+
173+function markdownBulletMatcher(input: string, position: number): number | null {
174+ if (lineOffset(input, position) === null) {
175+ return null;
176+ }
177+
178+ const single = input[position];
179+
180+ if (single === "*" || single === "+" || single === "-") {
181+ return /\s/.test(input[position + 1] ?? "") ? 1 : null;
182+ }
183+
184+ let cursor = position;
185+
186+ while (/\d/.test(input[cursor] ?? "")) {
187+ cursor += 1;
188+ }
189+
190+ if (cursor === position || input[cursor] !== ".") {
191+ return null;
192+ }
193+
194+ return /\s/.test(input[cursor + 1] ?? "") ? cursor - position + 1 : null;
195+}
196+
197+function markdownEmphasisMatcher(
198+ input: string,
199+ position: number,
200+): number | null {
201+ if (input[position] === "*" && input[position + 1] !== "*") {
202+ const end = findStyledSpanEnd(input, position, "*");
203+ return end === null ? null : end - position;
204+ }
205+
206+ if (input[position] === "_" && input[position + 1] !== "_") {
207+ const end = findStyledSpanEnd(input, position, "_");
208+ return end === null ? null : end - position;
209+ }
210+
211+ return null;
212+}
213+
214+function markdownHeadingMatcher(
215+ input: string,
216+ position: number,
217+): { children: InternalRule[]; length: number; scope: string } | null {
218+ if (lineOffset(input, position) === null || input[position] !== "#") {
219+ return null;
220+ }
221+
222+ let cursor = position;
223+
224+ while (input[cursor] === "#") {
225+ cursor += 1;
226+ }
227+
228+ if (cursor - position > 6) {
229+ return null;
230+ }
231+
232+ const next = input[cursor];
233+
234+ if (next && next !== "\n" && next !== " " && next !== "\t") {
235+ return null;
236+ }
237+
238+ return {
239+ children: MARKDOWN_INLINE_RULES,
240+ length: findLineEnd(input, position) - position,
241+ scope: "section",
242+ };
243+}
244+
245+function markdownInlineCodeMatcher(
246+ input: string,
247+ position: number,
248+): number | null {
249+ if (input[position] !== "`") {
250+ return null;
251+ }
252+
253+ const lineEnd = findLineEnd(input, position);
254+ const end = input.indexOf("`", position + 1);
255+
256+ if (end === -1 || end >= lineEnd || end === position + 1) {
257+ return null;
258+ }
259+
260+ return end - position + 1;
261+}
262+
263+function markdownLinkMatcher(
264+ input: string,
265+ position: number,
266+): { children: InternalRule[]; length: number } | null {
Cut short: this commit's diff is over 1000 lines.
src/languages/python.ts+46−0
Cut short: this commit's diff is over 1000 lines.
src/languages/typescript.ts+1−0
Cut short: this commit's diff is over 1000 lines.
test/lextide.test.ts+151−0
Cut short: this commit's diff is over 1000 lines.