719
Le
lextide / files
lextide / src / languages / helpers.ts
6.5 KB
1import type { InternalMatcher, InternalRule } from "../core/grammar"; 2 3const NUMBER_EXPRESSION = 4 /(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/; 5 6const ASCII_IDENTIFIER: IdentifierSpec = { 7 part: /^[A-Za-z0-9_]$/, 8 pattern: /[A-Za-z_][A-Za-z0-9_]*/, 9 start: /^[A-Za-z_]$/, 10}; 11 12const ECMASCRIPT_IDENTIFIER: IdentifierSpec = { 13 part: /^(?:[$_\u200C\u200D]|[\p{ID_Continue}])$/u, 14 pattern: /(?:[$_]|[\p{ID_Start}])(?:[$_\u200C\u200D]|[\p{ID_Continue}])*/u, 15 start: /^(?:[$_]|[\p{ID_Start}])$/u, 16}; 17 18const UNICODE_IDENTIFIER: IdentifierSpec = { 19 part: /^(?:_|[\p{XID_Continue}])$/u, 20 pattern: /(?:_|[\p{XID_Start}])(?:_|[\p{XID_Continue}])*/u, 21 start: /^(?:_|[\p{XID_Start}])$/u, 22}; 23 24interface IdentifierMatcherOptions { 25 builtInRelevance?: number; 26 builtIns?: ReadonlySet<string>; 27 identifier?: IdentifierSpec; 28 keywordRelevance?: number; 29 keywords?: ReadonlySet<string>; 30 literalRelevance?: number; 31 literals?: ReadonlySet<string>; 32} 33 34interface DeclarationRuleOptions { 35 allowGeneratorMarker?: boolean; 36 identifier?: IdentifierSpec; 37 relevance?: number; 38} 39 40export interface IdentifierSpec { 41 part: RegExp; 42 pattern: RegExp; 43 start: RegExp; 44} 45 46export { 47 ASCII_IDENTIFIER, 48 ECMASCRIPT_IDENTIFIER, 49 NUMBER_EXPRESSION, 50 UNICODE_IDENTIFIER, 51}; 52 53export function consumeUntil( 54 stopCharacters: readonly string[], 55): InternalMatcher { 56 const stops = new Set(stopCharacters); 57 58 return (input, position) => { 59 let cursor = position; 60 61 while (cursor < input.length && !stops.has(input[cursor])) { 62 cursor += codePointLength(input, cursor); 63 } 64 65 if (cursor === position) { 66 return null; 67 } 68 69 return cursor - position; 70 }; 71} 72 73export function createDeclarationRule( 74 keyword: string, 75 titleScope: string, 76 options: DeclarationRuleOptions = {}, 77): InternalRule { 78 const identifier = resolveIdentifierSpec(options.identifier); 79 80 return { 81 match(input, position) { 82 if (!startsWithWord(input, position, keyword)) { 83 return null; 84 } 85 86 const titleEnd = findDeclarationTitleEnd( 87 input, 88 position, 89 keyword, 90 options, 91 ); 92 93 if (titleEnd === null) { 94 return null; 95 } 96 97 return { 98 children: [ 99 { match: new RegExp(escapeForExpression(keyword)), scope: "keyword" }, 100 { match: identifier.pattern, scope: titleScope }, 101 ], 102 length: titleEnd - position, 103 relevance: options.relevance ?? 2, 104 }; 105 }, 106 }; 107} 108 109export function createIdentifierMatcher( 110 options: IdentifierMatcherOptions, 111): InternalMatcher { 112 const identifier = resolveIdentifierSpec(options.identifier); 113 114 return (input, position) => { 115 const end = scanIdentifier(input, position, identifier); 116 117 if (end === null) { 118 return null; 119 } 120 121 const value = input.slice(position, end); 122 123 if (options.keywords?.has(value)) { 124 return { 125 length: end - position, 126 relevance: options.keywordRelevance ?? 1, 127 scope: "keyword", 128 }; 129 } 130 131 if (options.literals?.has(value)) { 132 return { 133 length: end - position, 134 relevance: options.literalRelevance ?? 1, 135 scope: "literal", 136 }; 137 } 138 139 if (options.builtIns?.has(value)) { 140 return { 141 length: end - position, 142 relevance: options.builtInRelevance ?? 1, 143 scope: "built_in", 144 }; 145 } 146 147 return end - position; 148 }; 149} 150 151export function createNumberRule(relevance = 1): InternalRule { 152 return { 153 match: NUMBER_EXPRESSION, 154 relevance, 155 scope: "number", 156 }; 157} 158 159export function escapeForExpression(value: string): string { 160 return value.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"); 161} 162 163export function isIdentifierPart( 164 character: string, 165 identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER, 166): boolean { 167 return resolveIdentifierSpec(identifier).part.test(character); 168} 169 170export function isIdentifierStart( 171 character: string, 172 identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER, 173): boolean { 174 return resolveIdentifierSpec(identifier).start.test(character); 175} 176 177export function scanIdentifier( 178 input: string, 179 position: number, 180 identifier: IdentifierSpec | boolean = UNICODE_IDENTIFIER, 181): number | null { 182 const spec = resolveIdentifierSpec(identifier); 183 const start = readCodePoint(input, position); 184 185 if (!start || !spec.start.test(start.character)) { 186 return null; 187 } 188 189 let cursor = position + start.length; 190 191 while (cursor < input.length) { 192 const codePoint = readCodePoint(input, cursor); 193 194 if (!codePoint || !spec.part.test(codePoint.character)) { 195 break; 196 } 197 198 cursor += codePoint.length; 199 } 200 201 return cursor; 202} 203 204export function startsWithWord( 205 input: string, 206 position: number, 207 word: string, 208): boolean { 209 if (!input.startsWith(word, position)) { 210 return false; 211 } 212 213 const before = readCodePoint(input, previousCodePointIndex(input, position)); 214 const after = readCodePoint(input, position + word.length); 215 216 return ( 217 !isWordCharacter(before?.character ?? "") && 218 !isWordCharacter(after?.character ?? "") 219 ); 220} 221 222function codePointLength(input: string, position: number): number { 223 const codePoint = input.codePointAt(position); 224 225 if (codePoint === undefined) { 226 return 1; 227 } 228 229 return codePoint > 0xffff ? 2 : 1; 230} 231 232function findDeclarationTitleEnd( 233 input: string, 234 position: number, 235 keyword: string, 236 options: DeclarationRuleOptions, 237): number | null { 238 let cursor = skipWhitespace(input, position + keyword.length); 239 240 if (options.allowGeneratorMarker && input[cursor] === "*") { 241 cursor = skipWhitespace(input, cursor + 1); 242 } 243 244 return scanIdentifier(input, cursor, options.identifier); 245} 246 247function isWordCharacter(character: string): boolean { 248 return /^(?:[$_]|[\p{ID_Continue}]|[\p{XID_Continue}]|[\u200C\u200D])$/u.test( 249 character, 250 ); 251} 252 253function previousCodePointIndex(input: string, position: number): number { 254 if (position <= 0) { 255 return -1; 256 } 257 258 const previous = position - 1; 259 const code = input.charCodeAt(previous); 260 261 if (code >= 0xdc00 && code <= 0xdfff && previous > 0) { 262 return previous - 1; 263 } 264 265 return previous; 266} 267 268function readCodePoint( 269 input: string, 270 position: number, 271): { character: string; length: number } | null { 272 if (position < 0 || position >= input.length) { 273 return null; 274 } 275 276 const length = codePointLength(input, position); 277 278 return { 279 character: input.slice(position, position + length), 280 length, 281 }; 282} 283 284function resolveIdentifierSpec( 285 identifier: IdentifierSpec | boolean | undefined, 286): IdentifierSpec { 287 if (typeof identifier === "object") { 288 return identifier; 289 } 290 291 if (identifier === true) { 292 return ECMASCRIPT_IDENTIFIER; 293 } 294 295 if (identifier === false) { 296 return ASCII_IDENTIFIER; 297 } 298 299 return UNICODE_IDENTIFIER; 300} 301 302function skipWhitespace(input: string, position: number): number { 303 let cursor = position; 304 305 while (cursor < input.length && /\s/.test(input[cursor])) { 306 cursor += 1; 307 } 308 309 return cursor; 310}