@@ -0,0 +1,518 @@
1+import type { GrammarDefinition } from "../types";
2+import {
3+ getInternalStates,
4+ type InternalMatcher,
5+ type InternalRule,
6+ type MatchResult,
7+} from "./grammar";
8+
9+const stickyExpressions = new WeakMap<RegExp, RegExp>();
10+
11+interface ResolvedRule {
12+ children?: InternalRule[] | undefined;
13+ length: number;
14+ pop: boolean;
15+ push?: string | undefined;
16+ pushScope?: string | null | undefined;
17+ relevance: number;
18+ scope?: string | undefined;
19+}
20+
21+export interface Segment {
22+ text: string;
23+ scopes: string[];
24+ start: number;
25+ end: number;
26+ stateAfter: string[];
27+}
28+
29+export interface ParseResult {
30+ segments: Segment[];
31+ relevance: number;
32+ state: string[];
33+}
34+
35+export interface SegmentGroup {
36+ segments: Segment[];
37+ end: number;
38+ stateAfter: string[];
39+}
40+
41+export function parseSource(
42+ grammar: GrammarDefinition,
43+ input: string,
44+ initialState: readonly string[] = [grammar.startState],
45+): ParseResult {
46+ const states = getInternalStates(grammar);
47+ const stack =
48+ initialState.length > 0 ? [...initialState] : [grammar.startState];
49+ const segments: Segment[] = [];
50+ let relevance = 0;
51+ let position = 0;
52+
53+ while (position < input.length) {
54+ const state = currentState(states, stack);
55+ const resolved = matchStateRules(state.rules, input, position);
56+
57+ if (!resolved) {
58+ position = consumePlainSegment(states, stack, segments, input, position);
59+ continue;
60+ }
61+
62+ position = consumeResolvedSegment(
63+ states,
64+ stack,
65+ segments,
66+ input,
67+ position,
68+ resolved,
69+ );
70+ relevance += resolved.relevance;
71+ }
72+
73+ return { segments, relevance, state: stack };
74+}
75+
76+export function groupSegmentsByRootChild(
77+ segments: readonly Segment[],
78+): SegmentGroup[] {
79+ if (segments.length === 0) {
80+ return [];
81+ }
82+
83+ const groups: SegmentGroup[] = [];
84+ let currentKey = rootChildKey(segments[0]);
85+ let currentSegments: Segment[] = [segments[0]];
86+
87+ for (let index = 1; index < segments.length; index += 1) {
88+ const segment = segments[index];
89+ const key = rootChildKey(segment);
90+
91+ if (key === currentKey) {
92+ currentSegments.push(segment);
93+ continue;
94+ }
95+
96+ groups.push(createGroup(currentSegments));
97+ currentSegments = [segment];
98+ currentKey = key;
99+ }
100+
101+ groups.push(createGroup(currentSegments));
102+
103+ return groups;
104+}
105+
106+function consumePlainSegment(
107+ states: ReturnType<typeof getInternalStates>,
108+ stack: readonly string[],
109+ segments: Segment[],
110+ input: string,
111+ position: number,
112+): number {
113+ const end = position + codePointLength(input, position);
114+
115+ pushSegment(
116+ segments,
117+ input.slice(position, end),
118+ activeScopes(states, stack),
119+ position,
120+ end,
121+ stack,
122+ );
123+
124+ return end;
125+}
126+
127+function consumeResolvedSegment(
128+ states: ReturnType<typeof getInternalStates>,
129+ stack: string[],
130+ segments: Segment[],
131+ input: string,
132+ position: number,
133+ resolved: ResolvedRule,
134+): number {
135+ const end = position + resolved.length;
136+ const text = input.slice(position, end);
137+ const scopes = activeScopes(
138+ states,
139+ stack,
140+ resolved.pushScope,
141+ resolved.scope,
142+ );
143+
144+ emitResolvedSegments(segments, text, scopes, position, stack, resolved);
145+ applyStateTransition(stack, resolved);
146+ updateTailState(segments, stack);
147+
148+ return end;
149+}
150+
151+function currentState(
152+ states: ReturnType<typeof getInternalStates>,
153+ stack: readonly string[],
154+) {
155+ const stateName = stack[stack.length - 1];
156+ const state = states[stateName];
157+
158+ if (!state) {
159+ throw new Error(`Unknown grammar state: ${stateName}`);
160+ }
161+
162+ return state;
163+}
164+
165+function emitResolvedSegments(
166+ segments: Segment[],
167+ text: string,
168+ scopes: readonly string[],
169+ position: number,
170+ stack: readonly string[],
171+ resolved: ResolvedRule,
172+): void {
173+ if (resolved.children && resolved.children.length > 0) {
174+ const childSegments = parseChildren(
175+ text,
176+ resolved.children,
177+ scopes,
178+ position,
179+ stack,
180+ );
181+
182+ for (const childSegment of childSegments) {
183+ pushSegment(
184+ segments,
185+ childSegment.text,
186+ childSegment.scopes,
187+ childSegment.start,
188+ childSegment.end,
189+ stack,
190+ );
191+ }
192+
193+ return;
194+ }
195+
196+ pushSegment(
197+ segments,
198+ text,
199+ [...scopes],
200+ position,
201+ position + resolved.length,
202+ stack,
203+ );
204+}
205+
206+function applyStateTransition(stack: string[], resolved: ResolvedRule): void {
207+ if (resolved.push) {
208+ stack.push(resolved.push);
209+ }
210+
211+ if (resolved.pop && stack.length > 1) {
212+ stack.pop();
213+ }
214+}
215+
216+function updateTailState(segments: Segment[], stack: readonly string[]): void {
217+ const tail = segments[segments.length - 1];
218+
219+ if (tail) {
220+ tail.stateAfter = [...stack];
221+ }
222+}
223+
224+function activeScopes(
225+ states: ReturnType<typeof getInternalStates>,
226+ stack: readonly string[],
227+ pushScope?: string | null,
228+ transientScope?: string,
229+): string[] {
230+ const scopes = stack
231+ .map((stateName) => states[stateName]?.defaultScope)
232+ .filter((scope): scope is string => typeof scope === "string");
233+
234+ if (typeof pushScope === "string") {
235+ scopes.push(pushScope);
236+ }
237+
238+ if (typeof transientScope === "string") {
239+ scopes.push(transientScope);
240+ }
241+
242+ return scopes;
243+}
244+
245+function codePointLength(input: string, position: number): number {
246+ const codePoint = input.codePointAt(position);
247+
248+ if (codePoint === undefined) {
249+ return 1;
250+ }
251+
252+ return codePoint > 0xffff ? 2 : 1;
253+}
254+
255+function createGroup(segments: Segment[]): SegmentGroup {
256+ const tail = segments[segments.length - 1];
257+
258+ return {
259+ segments,
260+ end: tail.end,
261+ stateAfter: tail.stateAfter,
262+ };
263+}
264+
265+function matchExpression(
266+ expression: RegExp,
267+ input: string,
268+ position: number,
269+): string | null {
270+ const sticky = getStickyExpression(expression);
271+ sticky.lastIndex = position;
272+ const match = sticky.exec(input);
273+
274+ if (!match || match[0].length === 0) {
275+ return null;
276+ }
277+
278+ return match[0];
279+}
280+
281+function matchStateRules(
282+ rules: readonly InternalRule[],
283+ input: string,
284+ position: number,
285+): ResolvedRule | null {
286+ for (const rule of rules) {
287+ const resolved = resolveRule(rule, input, position);
288+
289+ if (resolved) {
290+ return resolved;
291+ }
292+ }
293+
294+ return null;
295+}
296+
297+function parseChildren(
298+ input: string,
299+ rules: readonly InternalRule[],
300+ baseScopes: readonly string[],
301+ offset: number,
302+ stateAfter: readonly string[],
303+): Segment[] {
304+ const segments: Segment[] = [];
305+ let position = 0;
306+
307+ while (position < input.length) {
308+ const resolved = matchStateRules(rules, input, position);
309+
310+ if (!resolved) {
311+ const length = codePointLength(input, position);
312+ const end = position + length;
313+
314+ pushSegment(
315+ segments,
316+ input.slice(position, end),
317+ [...baseScopes],
318+ offset + position,
319+ offset + end,
320+ stateAfter,
321+ );
322+
323+ position = end;
324+ continue;
325+ }
326+
327+ const end = position + resolved.length;
328+ const text = input.slice(position, end);
329+ const scopes = [...baseScopes];
330+
331+ if (resolved.scope) {
332+ scopes.push(resolved.scope);
333+ }
334+
335+ if (resolved.children && resolved.children.length > 0) {
336+ const childSegments = parseChildren(
337+ text,
338+ resolved.children,
339+ scopes,
340+ offset + position,
341+ stateAfter,
342+ );
343+
344+ for (const childSegment of childSegments) {
345+ pushSegment(
346+ segments,
347+ childSegment.text,
348+ childSegment.scopes,
349+ childSegment.start,
350+ childSegment.end,
351+ stateAfter,
352+ );
353+ }
354+ } else {
355+ pushSegment(
356+ segments,
357+ text,
358+ scopes,
359+ offset + position,
360+ offset + end,
361+ stateAfter,
362+ );
363+ }
364+
365+ position = end;
366+ }
367+
368+ return segments;
369+}
370+
371+function pushSegment(
372+ segments: Segment[],
373+ text: string,
374+ scopes: string[],
375+ start: number,
376+ end: number,
377+ stateAfter: readonly string[],
378+): void {
379+ if (text.length === 0) {
380+ return;
381+ }
382+
383+ const tail = segments[segments.length - 1];
384+
385+ if (tail && tail.end === start && sameScopes(tail.scopes, scopes)) {
386+ tail.text += text;
387+ tail.end = end;
388+ tail.stateAfter = [...stateAfter];
389+ return;
390+ }
391+
392+ segments.push({
393+ text,
394+ scopes,
395+ start,
396+ end,
397+ stateAfter: [...stateAfter],
398+ });
399+}
400+
401+function resolveRule(
402+ rule: InternalRule,
403+ input: string,
404+ position: number,
405+): ResolvedRule | null {
406+ const matched = matchRule(rule, input, position);
407+
408+ if (!isValidMatch(matched)) {
409+ return null;
410+ }
411+
412+ if (typeof matched === "object" && matched !== null) {
413+ return buildResolvedRuleFromObject(rule, matched);
414+ }
415+
416+ return buildResolvedRuleFromLength(rule, matchLength(matched));
417+}
418+
419+function matchRule(
420+ rule: InternalRule,
421+ input: string,
422+ position: number,
423+): MatchResult | string {
424+ return rule.match instanceof RegExp
425+ ? matchExpression(rule.match, input, position)
426+ : resolveMatcher(rule.match, input, position);
427+}
428+
429+function resolveMatcher(
430+ matcher: InternalMatcher,
431+ input: string,
432+ position: number,
433+): MatchResult {
434+ return matcher(input, position);
Cut short: this commit's diff is over 1000 lines.