719
Le
lextide / files
lextide / src / languages / python.ts
7.4 KB
1import type { InternalRule } from "../core/grammar"; 2import { defineGrammar } from "../core/grammar"; 3import type { LanguageFn } from "../types"; 4 5import { 6 consumeUntil, 7 createDeclarationRule, 8 createIdentifierMatcher, 9 createNumberRule, 10 scanIdentifier, 11 UNICODE_IDENTIFIER, 12} from "./helpers"; 13 14const PYTHON_KEYWORDS = new Set([ 15 "and", 16 "as", 17 "assert", 18 "async", 19 "await", 20 "break", 21 "case", 22 "class", 23 "continue", 24 "def", 25 "del", 26 "elif", 27 "else", 28 "except", 29 "finally", 30 "for", 31 "from", 32 "global", 33 "if", 34 "import", 35 "in", 36 "is", 37 "lambda", 38 "match", 39 "nonlocal", 40 "not", 41 "or", 42 "pass", 43 "raise", 44 "return", 45 "try", 46 "while", 47 "with", 48 "yield", 49]); 50 51const PYTHON_BUILT_INS = new Set([ 52 "abs", 53 "all", 54 "any", 55 "bool", 56 "bytes", 57 "dict", 58 "enumerate", 59 "filter", 60 "float", 61 "help", 62 "input", 63 "int", 64 "isinstance", 65 "len", 66 "list", 67 "map", 68 "max", 69 "min", 70 "next", 71 "object", 72 "open", 73 "print", 74 "property", 75 "range", 76 "repr", 77 "reversed", 78 "set", 79 "slice", 80 "sorted", 81 "str", 82 "sum", 83 "super", 84 "tuple", 85 "type", 86 "zip", 87]); 88 89const PYTHON_LITERALS = new Set([ 90 "Ellipsis", 91 "False", 92 "None", 93 "NotImplemented", 94 "True", 95]); 96 97const PYTHON_FORMAT_PREFIX = /(?:[fFtT]|[fFtT][rR]|[rR][fFtT])/; 98const PYTHON_PLAIN_PREFIX = /(?:[bBuUrR]|[bB][rR]|[rR][bB])?/; 99const PYTHON_IDENTIFIER_RULE = { 100 match: createIdentifierMatcher({ 101 builtInRelevance: 1, 102 builtIns: PYTHON_BUILT_INS, 103 identifier: UNICODE_IDENTIFIER, 104 keywordRelevance: 2, 105 keywords: PYTHON_KEYWORDS, 106 literalRelevance: 1, 107 literals: PYTHON_LITERALS, 108 }), 109}; 110 111export const python: LanguageFn = () => 112 defineGrammar({ 113 aliases: ["py"], 114 name: "python", 115 relevance: 0, 116 startState: "root", 117 states: { 118 doubleString: { 119 defaultScope: "string", 120 rules: [ 121 { match: /\\./, scope: "escape" }, 122 { match: /\n/, pop: true }, 123 { match: /"/, pop: true }, 124 { match: consumeUntil(["\\", '"', "\n"]) }, 125 ], 126 }, 127 formatSubst: { 128 defaultScope: "subst", 129 rules: createFormatSubstRules(), 130 }, 131 formattedDoubleString: { 132 defaultScope: "string", 133 rules: createFormattedStringRules( 134 /"/, 135 ["\\", '"', "{", "}", "\n"], 136 true, 137 ), 138 }, 139 formattedSingleString: { 140 defaultScope: "string", 141 rules: createFormattedStringRules( 142 /'/, 143 ["\\", "'", "{", "}", "\n"], 144 true, 145 ), 146 }, 147 formattedTripleDoubleString: { 148 defaultScope: "string", 149 rules: createFormattedStringRules(/"""/, ["\\", '"', "{", "}"]), 150 }, 151 formattedTripleSingleString: { 152 defaultScope: "string", 153 rules: createFormattedStringRules(/'''/, ["\\", "'", "{", "}"]), 154 }, 155 root: { 156 rules: [ 157 { match: /#[^\n]*/, scope: "comment", relevance: 1 }, 158 { 159 match: pythonDecoratorMatcher, 160 scope: "meta", 161 relevance: 1, 162 }, 163 { 164 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}"""`), 165 push: "formattedTripleDoubleString", 166 pushScope: "string", 167 relevance: 2, 168 }, 169 { 170 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}'''`), 171 push: "formattedTripleSingleString", 172 pushScope: "string", 173 relevance: 2, 174 }, 175 { 176 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}"""`), 177 push: "tripleDoubleString", 178 pushScope: "string", 179 relevance: 2, 180 }, 181 { 182 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}'''`), 183 push: "tripleSingleString", 184 pushScope: "string", 185 relevance: 2, 186 }, 187 { 188 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}"`), 189 push: "formattedDoubleString", 190 pushScope: "string", 191 relevance: 1, 192 }, 193 { 194 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}'`), 195 push: "formattedSingleString", 196 pushScope: "string", 197 relevance: 1, 198 }, 199 { 200 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}"`), 201 push: "doubleString", 202 pushScope: "string", 203 relevance: 1, 204 }, 205 { 206 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}'`), 207 push: "singleString", 208 pushScope: "string", 209 relevance: 1, 210 }, 211 createDeclarationRule("def", "title.function", { 212 identifier: UNICODE_IDENTIFIER, 213 relevance: 3, 214 }), 215 createDeclarationRule("class", "title.class", { 216 identifier: UNICODE_IDENTIFIER, 217 relevance: 2, 218 }), 219 createNumberRule(), 220 PYTHON_IDENTIFIER_RULE, 221 ], 222 }, 223 singleString: { 224 defaultScope: "string", 225 rules: [ 226 { match: /\\./, scope: "escape" }, 227 { match: /\n/, pop: true }, 228 { match: /'/, pop: true }, 229 { match: consumeUntil(["\\", "'", "\n"]) }, 230 ], 231 }, 232 tripleDoubleString: { 233 defaultScope: "string", 234 rules: [ 235 { match: /\\./, scope: "escape" }, 236 { match: /"""/, pop: true }, 237 { match: consumeUntil(["\\", '"']) }, 238 ], 239 }, 240 tripleSingleString: { 241 defaultScope: "string", 242 rules: [ 243 { match: /\\./, scope: "escape" }, 244 { match: /'''/, pop: true }, 245 { match: consumeUntil(["\\", "'"]) }, 246 ], 247 }, 248 }, 249 }); 250 251function pythonDecoratorMatcher( 252 input: string, 253 position: number, 254): number | null { 255 if (input[position] !== "@" || !isDecoratorStart(input, position)) { 256 return null; 257 } 258 259 let cursor = position + 1; 260 const first = scanIdentifier(input, cursor, UNICODE_IDENTIFIER); 261 262 if (first === null) { 263 return null; 264 } 265 266 cursor = first; 267 268 while (input[cursor] === ".") { 269 const next = scanIdentifier(input, cursor + 1, UNICODE_IDENTIFIER); 270 271 if (next === null) { 272 return null; 273 } 274 275 cursor = next; 276 } 277 278 return cursor - position; 279} 280 281function isDecoratorStart(input: string, position: number): boolean { 282 let cursor = position - 1; 283 284 while (cursor >= 0 && (input[cursor] === " " || input[cursor] === "\t")) { 285 cursor -= 1; 286 } 287 288 return cursor < 0 || input[cursor] === "\n"; 289} 290 291function createFormatSubstRules(): InternalRule[] { 292 return [ 293 { 294 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}"""`), 295 push: "formattedTripleDoubleString", 296 pushScope: "string", 297 relevance: 1, 298 }, 299 { 300 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}'''`), 301 push: "formattedTripleSingleString", 302 pushScope: "string", 303 relevance: 1, 304 }, 305 { 306 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}"""`), 307 push: "tripleDoubleString", 308 pushScope: "string", 309 relevance: 1, 310 }, 311 { 312 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}'''`), 313 push: "tripleSingleString", 314 pushScope: "string", 315 relevance: 1, 316 }, 317 { 318 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}"`), 319 push: "formattedDoubleString", 320 pushScope: "string", 321 relevance: 1, 322 }, 323 { 324 match: new RegExp(`${PYTHON_FORMAT_PREFIX.source}'`), 325 push: "formattedSingleString", 326 pushScope: "string", 327 relevance: 1, 328 }, 329 { 330 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}"`), 331 push: "doubleString", 332 pushScope: "string", 333 relevance: 1, 334 }, 335 { 336 match: new RegExp(`${PYTHON_PLAIN_PREFIX.source}'`), 337 push: "singleString", 338 pushScope: "string", 339 relevance: 1, 340 }, 341 { match: /\{/, push: "formatSubst", relevance: 0 }, 342 { match: /\}/, pop: true, relevance: 0 }, 343 createNumberRule(), 344 PYTHON_IDENTIFIER_RULE, 345 ]; 346} 347 348function createFormattedStringRules( 349 end: RegExp, 350 stopCharacters: readonly string[], 351 stopOnNewline = false, 352): InternalRule[] { 353 const rules: InternalRule[] = [ 354 { match: /\\./, scope: "escape" }, 355 { match: /\{\{|\}\}/, relevance: 0 }, 356 { match: /\{/, push: "formatSubst", pushScope: "subst", relevance: 1 }, 357 ]; 358 359 if (stopOnNewline) { 360 rules.push({ match: /\n/, pop: true }); 361 } 362 363 rules.push({ match: end, pop: true }); 364 rules.push({ match: consumeUntil(stopCharacters) }); 365 366 return rules; 367}