7814
Ta
tiny-c-agent / files
tiny-c-agent / vendor / typescript / common / scanner.h
9.9 KB
1#include "tree_sitter/parser.h" 2 3#include <wctype.h> 4 5enum TokenType { 6 AUTOMATIC_SEMICOLON, 7 TEMPLATE_CHARS, 8 TERNARY_QMARK, 9 HTML_COMMENT, 10 LOGICAL_OR, 11 ESCAPE_SEQUENCE, 12 REGEX_PATTERN, 13 JSX_TEXT, 14 FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON, 15 ERROR_RECOVERY, 16}; 17 18static void advance(TSLexer *lexer) { lexer->advance(lexer, false); } 19 20static void skip(TSLexer *lexer) { lexer->advance(lexer, true); } 21 22static bool scan_template_chars(TSLexer *lexer) { 23 lexer->result_symbol = TEMPLATE_CHARS; 24 for (bool has_content = false;; has_content = true) { 25 lexer->mark_end(lexer); 26 switch (lexer->lookahead) { 27 case '`': 28 return has_content; 29 case '\0': 30 return false; 31 case '$': 32 advance(lexer); 33 if (lexer->lookahead == '{') { 34 return has_content; 35 } 36 break; 37 case '\\': 38 return has_content; 39 default: 40 advance(lexer); 41 } 42 } 43} 44 45static bool scan_whitespace_and_comments(TSLexer *lexer, bool *scanned_comment) { 46 for (;;) { 47 while (iswspace(lexer->lookahead)) { 48 skip(lexer); 49 } 50 51 if (lexer->lookahead == '/') { 52 skip(lexer); 53 54 if (lexer->lookahead == '/') { 55 skip(lexer); 56 while (lexer->lookahead != 0 && lexer->lookahead != '\n') { 57 skip(lexer); 58 } 59 *scanned_comment = true; 60 } else if (lexer->lookahead == '*') { 61 skip(lexer); 62 while (lexer->lookahead != 0) { 63 if (lexer->lookahead == '*') { 64 skip(lexer); 65 if (lexer->lookahead == '/') { 66 skip(lexer); 67 break; 68 } 69 } else { 70 skip(lexer); 71 } 72 } 73 } else { 74 return false; 75 } 76 } else { 77 return true; 78 } 79 } 80} 81 82static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols, bool *scanned_comment) { 83 lexer->result_symbol = AUTOMATIC_SEMICOLON; 84 lexer->mark_end(lexer); 85 86 for (;;) { 87 if (lexer->lookahead == 0) { 88 return true; 89 } 90 if (lexer->lookahead == '}') { 91 // Automatic semicolon insertion breaks detection of object patterns 92 // in a typed context: 93 // type F = ({a}: {a: number}) => number; 94 // Therefore, disable automatic semicolons when followed by typing 95 do { 96 skip(lexer); 97 } while (iswspace(lexer->lookahead)); 98 if (lexer->lookahead == ':') { 99 return valid_symbols[LOGICAL_OR]; // Don't return false if we're in a ternary by checking if || is valid 100 } 101 return true; 102 } 103 if (!iswspace(lexer->lookahead)) { 104 return false; 105 } 106 if (lexer->lookahead == '\n') { 107 break; 108 } 109 skip(lexer); 110 } 111 112 skip(lexer); 113 114 if (!scan_whitespace_and_comments(lexer, scanned_comment)) { 115 return false; 116 } 117 118 switch (lexer->lookahead) { 119 case '`': 120 case ',': 121 case '.': 122 case ';': 123 case '*': 124 case '%': 125 case '>': 126 case '<': 127 case '=': 128 case '?': 129 case '^': 130 case '|': 131 case '&': 132 case '/': 133 case ':': 134 return false; 135 136 case '{': 137 if (valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) { 138 return false; 139 } 140 break; 141 142 // Don't insert a semicolon before a '[' or '(', unless we're parsing 143 // a type. Detect whether we're parsing a type or an expression using 144 // the validity of a binary operator token. 145 case '(': 146 case '[': 147 if (valid_symbols[LOGICAL_OR]) { 148 return false; 149 } 150 break; 151 152 // Insert a semicolon before `--` and `++`, but not before binary `+` or `-`. 153 case '+': 154 skip(lexer); 155 return lexer->lookahead == '+'; 156 case '-': 157 skip(lexer); 158 return lexer->lookahead == '-'; 159 160 // Don't insert a semicolon before `!=`, but do insert one before a unary `!`. 161 case '!': 162 skip(lexer); 163 return lexer->lookahead != '='; 164 165 // Don't insert a semicolon before `in` or `instanceof`, but do insert one 166 // before an identifier. 167 case 'i': 168 skip(lexer); 169 170 if (lexer->lookahead != 'n') { 171 return true; 172 } 173 skip(lexer); 174 175 if (!iswalpha(lexer->lookahead)) { 176 return false; 177 } 178 179 for (unsigned i = 0; i < 8; i++) { 180 if (lexer->lookahead != "stanceof"[i]) { 181 return true; 182 } 183 skip(lexer); 184 } 185 186 if (!iswalpha(lexer->lookahead)) { 187 return false; 188 } 189 break; 190 } 191 192 return true; 193} 194 195static bool scan_ternary_qmark(TSLexer *lexer) { 196 for (;;) { 197 if (!iswspace(lexer->lookahead)) { 198 break; 199 } 200 skip(lexer); 201 } 202 203 if (lexer->lookahead == '?') { 204 advance(lexer); 205 206 /* Optional chaining. */ 207 if (lexer->lookahead == '?' || lexer->lookahead == '.') { 208 return false; 209 } 210 211 lexer->mark_end(lexer); 212 lexer->result_symbol = TERNARY_QMARK; 213 214 /* TypeScript optional arguments contain the ?: sequence, possibly 215 with whitespace. */ 216 for (;;) { 217 if (!iswspace(lexer->lookahead)) { 218 break; 219 } 220 advance(lexer); 221 } 222 223 if (lexer->lookahead == ':' || lexer->lookahead == ')' || lexer->lookahead == ',') { 224 return false; 225 } 226 227 if (lexer->lookahead == '.') { 228 advance(lexer); 229 if (iswdigit(lexer->lookahead)) { 230 return true; 231 } 232 return false; 233 } 234 return true; 235 } 236 return false; 237} 238 239static bool scan_closing_comment(TSLexer *lexer) { 240 while (iswspace(lexer->lookahead) || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) { 241 skip(lexer); 242 } 243 244 const char *comment_start = "<!--"; 245 const char *comment_end = "-->"; 246 247 if (lexer->lookahead == '<') { 248 for (unsigned i = 0; i < 4; i++) { 249 if (lexer->lookahead != comment_start[i]) { 250 return false; 251 } 252 advance(lexer); 253 } 254 } else if (lexer->lookahead == '-') { 255 for (unsigned i = 0; i < 3; i++) { 256 if (lexer->lookahead != comment_end[i]) { 257 return false; 258 } 259 advance(lexer); 260 } 261 } else { 262 return false; 263 } 264 265 while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 && 266 lexer->lookahead != 0x2029) { 267 advance(lexer); 268 } 269 270 lexer->result_symbol = HTML_COMMENT; 271 lexer->mark_end(lexer); 272 273 return true; 274} 275 276static bool scan_jsx_text(TSLexer *lexer) { 277 // saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and 278 // does not immediately follow a newline. 279 bool saw_text = false; 280 // at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but 281 // immediately follows a newline. 282 bool at_newline = false; 283 284 while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' && 285 lexer->lookahead != '}' && lexer->lookahead != '&') { 286 bool is_wspace = iswspace(lexer->lookahead); 287 if (lexer->lookahead == '\n') { 288 at_newline = true; 289 } else { 290 // If at_newline is already true, and we see some whitespace, then it must stay true. 291 // Otherwise, it should be false. 292 // 293 // See the table below to determine the logic for computing `saw_text`. 294 // 295 // |------------------------------------| 296 // | at_newline | is_wspace | saw_text | 297 // |------------|-----------|-----------| 298 // | false (0) | false (0) | true (1) | 299 // | false (0) | true (1) | true (1) | 300 // | true (1) | false (0) | true (1) | 301 // | true (1) | true (1) | false (0) | 302 // |------------------------------------| 303 304 at_newline &= is_wspace; 305 if (!at_newline) { 306 saw_text = true; 307 } 308 } 309 310 advance(lexer); 311 } 312 313 lexer->result_symbol = JSX_TEXT; 314 return saw_text; 315} 316 317static inline bool external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { 318 if (valid_symbols[TEMPLATE_CHARS]) { 319 if (valid_symbols[AUTOMATIC_SEMICOLON]) { 320 return false; 321 } 322 return scan_template_chars(lexer); 323 } 324 325 if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) { 326 return true; 327 } 328 329 if (valid_symbols[AUTOMATIC_SEMICOLON] || valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) { 330 bool scanned_comment = false; 331 bool ret = scan_automatic_semicolon(lexer, valid_symbols, &scanned_comment); 332 if (!ret && !scanned_comment && valid_symbols[TERNARY_QMARK] && lexer->lookahead == '?') { 333 return scan_ternary_qmark(lexer); 334 } 335 return ret; 336 } 337 if (valid_symbols[TERNARY_QMARK]) { 338 return scan_ternary_qmark(lexer); 339 } 340 341 if (valid_symbols[HTML_COMMENT] && !valid_symbols[LOGICAL_OR] && !valid_symbols[ESCAPE_SEQUENCE] && 342 !valid_symbols[REGEX_PATTERN]) { 343 return scan_closing_comment(lexer); 344 } 345 346 return false; 347}