xonecas.io
719
Le

lextide / files

lextide / src / languages / helpers.ts

6.5 KB
1import type { InternalMatcher, InternalRule } from "../core/grammar";
2
3const NUMBER_EXPRESSION =
4	/(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/;
5
6const ASCII_IDENTIFIER: IdentifierSpec = {
7	part: /^[A-Za-z0-9_]$/,
8	pattern: /[A-Za-z_][A-Za-z0-9_]*/,
9	start: /^[A-Za-z_]$/,
10};
11
12const ECMASCRIPT_IDENTIFIER: IdentifierSpec = {
13	part: /^(?:[$_\u200C\u200D]|[\p{ID_Continue}])$/u,
14	pattern: /(?:[$_]|[\p{ID_Start}])(?:[$_\u200C\u200D]|[\p{ID_Continue}])*/u,
15	start: /^(?:[$_]|[\p{ID_Start}])$/u,
16};
17
18const UNICODE_IDENTIFIER: IdentifierSpec = {
19	part: /^(?:_|[\p{XID_Continue}])$/u,
20	pattern: /(?:_|[\p{XID_Start}])(?:_|[\p{XID_Continue}])*/u,
21	start: /^(?:_|[\p{XID_Start}])$/u,
22};
23
24interface IdentifierMatcherOptions {
25	builtInRelevance?: number;
26	builtIns?: ReadonlySet<string>;
27	identifier?: IdentifierSpec;
28	keywordRelevance?: number;
29	keywords?: ReadonlySet<string>;
30	literalRelevance?: number;
31	literals?: ReadonlySet<string>;
32}
33
34interface DeclarationRuleOptions {
35	allowGeneratorMarker?: boolean;
36	identifier?: IdentifierSpec;
37	relevance?: number;
38}
39
40export interface IdentifierSpec {
41	part: RegExp;
42	pattern: RegExp;
43	start: RegExp;
44}
45
46export {
47	ASCII_IDENTIFIER,
48	ECMASCRIPT_IDENTIFIER,
49	NUMBER_EXPRESSION,
50	UNICODE_IDENTIFIER,
51};
52
53export function consumeUntil(
54	stopCharacters: readonly string[],
55): InternalMatcher {
56	const stops = new Set(stopCharacters);
57
58	return (input, position) => {
59		let cursor = position;
60
61		while (cursor < input.length && !stops.has(input[cursor])) {
62			cursor += codePointLength(input, cursor);
63		}
64
65		if (cursor === position) {
66			return null;
67		}
68
69		return cursor - position;
70	};
71}
72
73export function createDeclarationRule(
74	keyword: string,
75	titleScope: string,
76	options: DeclarationRuleOptions = {},
77): InternalRule {
78	const identifier = resolveIdentifierSpec(options.identifier);
79
80	return {
81		match(input, position) {
82			if (!startsWithWord(input, position, keyword)) {
83				return null;
84			}
85
86			const titleEnd = findDeclarationTitleEnd(
87				input,
88				position,
89				keyword,
90				options,
91			);
92
93			if (titleEnd === null) {
94				return null;
95			}
96
97			return {
98				children: [
99					{ match: new RegExp(escapeForExpression(keyword)), scope: "keyword" },
100					{ match: identifier.pattern, scope: titleScope },
101				],
102				length: titleEnd - position,
103				relevance: options.relevance ?? 2,
104			};
105		},
106	};
107}
108
109export function createIdentifierMatcher(
110	options: IdentifierMatcherOptions,
111): InternalMatcher {
112	const identifier = resolveIdentifierSpec(options.identifier);
113
114	return (input, position) => {
115		const end = scanIdentifier(input, position, identifier);
116
117		if (end === null) {
118			return null;
119		}
120
121		const value = input.slice(position, end);
122
123		if (options.keywords?.has(value)) {
124			return {
125				length: end - position,
126				relevance: options.keywordRelevance ?? 1,
127				scope: "keyword",
128			};
129		}
130
131		if (options.literals?.has(value)) {
132			return {
133				length: end - position,
134				relevance: options.literalRelevance ?? 1,
135				scope: "literal",
136			};
137		}
138
139		if (options.builtIns?.has(value)) {
140			return {
141				length: end - position,
142				relevance: options.builtInRelevance ?? 1,
143				scope: "built_in",
144			};
145		}
146
147		return end - position;
148	};
149}
150
151export function createNumberRule(relevance = 1): InternalRule {
152	return {
153		match: NUMBER_EXPRESSION,
154		relevance,
155		scope: "number",
156	};
157}
158
159export function escapeForExpression(value: string): string {
160	return value.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
161}
162
163export function isIdentifierPart(
164	character: string,
165	identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER,
166): boolean {
167	return resolveIdentifierSpec(identifier).part.test(character);
168}
169
170export function isIdentifierStart(
171	character: string,
172	identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER,
173): boolean {
174	return resolveIdentifierSpec(identifier).start.test(character);
175}
176
177export function scanIdentifier(
178	input: string,
179	position: number,
180	identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER,
181): number | null {
182	const spec = resolveIdentifierSpec(identifier);
183	const start = readCodePoint(input, position);
184
185	if (!start || !spec.start.test(start.character)) {
186		return null;
187	}
188
189	let cursor = position + start.length;
190
191	while (cursor < input.length) {
192		const codePoint = readCodePoint(input, cursor);
193
194		if (!codePoint || !spec.part.test(codePoint.character)) {
195			break;
196		}
197
198		cursor += codePoint.length;
199	}
200
201	return cursor;
202}
203
204export function startsWithWord(
205	input: string,
206	position: number,
207	word: string,
208): boolean {
209	if (!input.startsWith(word, position)) {
210		return false;
211	}
212
213	const before = readCodePoint(input, previousCodePointIndex(input, position));
214	const after = readCodePoint(input, position + word.length);
215
216	return (
217		!isWordCharacter(before?.character ?? "") &&
218		!isWordCharacter(after?.character ?? "")
219	);
220}
221
222function codePointLength(input: string, position: number): number {
223	const codePoint = input.codePointAt(position);
224
225	if (codePoint === undefined) {
226		return 1;
227	}
228
229	return codePoint > 0xffff ? 2 : 1;
230}
231
232function findDeclarationTitleEnd(
233	input: string,
234	position: number,
235	keyword: string,
236	options: DeclarationRuleOptions,
237): number | null {
238	let cursor = skipWhitespace(input, position + keyword.length);
239
240	if (options.allowGeneratorMarker && input[cursor] === "*") {
241		cursor = skipWhitespace(input, cursor + 1);
242	}
243
244	return scanIdentifier(input, cursor, options.identifier);
245}
246
247function isWordCharacter(character: string): boolean {
248	return /^(?:[$_]|[\p{ID_Continue}]|[\p{XID_Continue}]|[\u200C\u200D])$/u.test(
249		character,
250	);
251}
252
253function previousCodePointIndex(input: string, position: number): number {
254	if (position <= 0) {
255		return -1;
256	}
257
258	const previous = position - 1;
259	const code = input.charCodeAt(previous);
260
261	if (code >= 0xdc00 && code <= 0xdfff && previous > 0) {
262		return previous - 1;
263	}
264
265	return previous;
266}
267
268function readCodePoint(
269	input: string,
270	position: number,
271): { character: string; length: number } | null {
272	if (position < 0 || position >= input.length) {
273		return null;
274	}
275
276	const length = codePointLength(input, position);
277
278	return {
279		character: input.slice(position, position + length),
280		length,
281	};
282}
283
284function resolveIdentifierSpec(
285	identifier: IdentifierSpec | boolean | undefined,
286): IdentifierSpec {
287	if (typeof identifier === "object") {
288		return identifier;
289	}
290
291	if (identifier === true) {
292		return ECMASCRIPT_IDENTIFIER;
293	}
294
295	if (identifier === false) {
296		return ASCII_IDENTIFIER;
297	}
298
299	return UNICODE_IDENTIFIER;
300}
301
302function skipWhitespace(input: string, position: number): number {
303	let cursor = position;
304
305	while (cursor < input.length && /\s/.test(input[cursor])) {
306		cursor += 1;
307	}
308
309	return cursor;
310}