xonecas.io
7814
Ta

tiny-c-agent / files

tiny-c-agent / vendor / typescript / common / scanner.h

9.9 KB
1#include "tree_sitter/parser.h"
2
3#include <wctype.h>
4
5enum TokenType {
6    AUTOMATIC_SEMICOLON,
7    TEMPLATE_CHARS,
8    TERNARY_QMARK,
9    HTML_COMMENT,
10    LOGICAL_OR,
11    ESCAPE_SEQUENCE,
12    REGEX_PATTERN,
13    JSX_TEXT,
14    FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON,
15    ERROR_RECOVERY,
16};
17
18static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
19
20static void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
21
22static bool scan_template_chars(TSLexer *lexer) {
23    lexer->result_symbol = TEMPLATE_CHARS;
24    for (bool has_content = false;; has_content = true) {
25        lexer->mark_end(lexer);
26        switch (lexer->lookahead) {
27            case '`':
28                return has_content;
29            case '\0':
30                return false;
31            case '$':
32                advance(lexer);
33                if (lexer->lookahead == '{') {
34                    return has_content;
35                }
36                break;
37            case '\\':
38                return has_content;
39            default:
40                advance(lexer);
41        }
42    }
43}
44
45static bool scan_whitespace_and_comments(TSLexer *lexer, bool *scanned_comment) {
46    for (;;) {
47        while (iswspace(lexer->lookahead)) {
48            skip(lexer);
49        }
50
51        if (lexer->lookahead == '/') {
52            skip(lexer);
53
54            if (lexer->lookahead == '/') {
55                skip(lexer);
56                while (lexer->lookahead != 0 && lexer->lookahead != '\n') {
57                    skip(lexer);
58                }
59                *scanned_comment = true;
60            } else if (lexer->lookahead == '*') {
61                skip(lexer);
62                while (lexer->lookahead != 0) {
63                    if (lexer->lookahead == '*') {
64                        skip(lexer);
65                        if (lexer->lookahead == '/') {
66                            skip(lexer);
67                            break;
68                        }
69                    } else {
70                        skip(lexer);
71                    }
72                }
73            } else {
74                return false;
75            }
76        } else {
77            return true;
78        }
79    }
80}
81
82static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols, bool *scanned_comment) {
83    lexer->result_symbol = AUTOMATIC_SEMICOLON;
84    lexer->mark_end(lexer);
85
86    for (;;) {
87        if (lexer->lookahead == 0) {
88            return true;
89        }
90        if (lexer->lookahead == '}') {
91            // Automatic semicolon insertion breaks detection of object patterns
92            // in a typed context:
93            //   type F = ({a}: {a: number}) => number;
94            // Therefore, disable automatic semicolons when followed by typing
95            do {
96                skip(lexer);
97            } while (iswspace(lexer->lookahead));
98            if (lexer->lookahead == ':') {
99                return valid_symbols[LOGICAL_OR]; // Don't return false if we're in a ternary by checking if || is valid
100            }
101            return true;
102        }
103        if (!iswspace(lexer->lookahead)) {
104            return false;
105        }
106        if (lexer->lookahead == '\n') {
107            break;
108        }
109        skip(lexer);
110    }
111
112    skip(lexer);
113
114    if (!scan_whitespace_and_comments(lexer, scanned_comment)) {
115        return false;
116    }
117
118    switch (lexer->lookahead) {
119        case '`':
120        case ',':
121        case '.':
122        case ';':
123        case '*':
124        case '%':
125        case '>':
126        case '<':
127        case '=':
128        case '?':
129        case '^':
130        case '|':
131        case '&':
132        case '/':
133        case ':':
134            return false;
135
136        case '{':
137            if (valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
138                return false;
139            }
140            break;
141
142            // Don't insert a semicolon before a '[' or '(', unless we're parsing
143            // a type. Detect whether we're parsing a type or an expression using
144            // the validity of a binary operator token.
145        case '(':
146        case '[':
147            if (valid_symbols[LOGICAL_OR]) {
148                return false;
149            }
150            break;
151
152            // Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
153        case '+':
154            skip(lexer);
155            return lexer->lookahead == '+';
156        case '-':
157            skip(lexer);
158            return lexer->lookahead == '-';
159
160            // Don't insert a semicolon before `!=`, but do insert one before a unary `!`.
161        case '!':
162            skip(lexer);
163            return lexer->lookahead != '=';
164
165            // Don't insert a semicolon before `in` or `instanceof`, but do insert one
166            // before an identifier.
167        case 'i':
168            skip(lexer);
169
170            if (lexer->lookahead != 'n') {
171                return true;
172            }
173            skip(lexer);
174
175            if (!iswalpha(lexer->lookahead)) {
176                return false;
177            }
178
179            for (unsigned i = 0; i < 8; i++) {
180                if (lexer->lookahead != "stanceof"[i]) {
181                    return true;
182                }
183                skip(lexer);
184            }
185
186            if (!iswalpha(lexer->lookahead)) {
187                return false;
188            }
189            break;
190    }
191
192    return true;
193}
194
195static bool scan_ternary_qmark(TSLexer *lexer) {
196    for (;;) {
197        if (!iswspace(lexer->lookahead)) {
198            break;
199        }
200        skip(lexer);
201    }
202
203    if (lexer->lookahead == '?') {
204        advance(lexer);
205
206        /* Optional chaining. */
207        if (lexer->lookahead == '?' || lexer->lookahead == '.') {
208            return false;
209        }
210
211        lexer->mark_end(lexer);
212        lexer->result_symbol = TERNARY_QMARK;
213
214        /* TypeScript optional arguments contain the ?: sequence, possibly
215           with whitespace. */
216        for (;;) {
217            if (!iswspace(lexer->lookahead)) {
218                break;
219            }
220            advance(lexer);
221        }
222
223        if (lexer->lookahead == ':' || lexer->lookahead == ')' || lexer->lookahead == ',') {
224            return false;
225        }
226
227        if (lexer->lookahead == '.') {
228            advance(lexer);
229            if (iswdigit(lexer->lookahead)) {
230                return true;
231            }
232            return false;
233        }
234        return true;
235    }
236    return false;
237}
238
239static bool scan_closing_comment(TSLexer *lexer) {
240    while (iswspace(lexer->lookahead) || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
241        skip(lexer);
242    }
243
244    const char *comment_start = "<!--";
245    const char *comment_end = "-->";
246
247    if (lexer->lookahead == '<') {
248        for (unsigned i = 0; i < 4; i++) {
249            if (lexer->lookahead != comment_start[i]) {
250                return false;
251            }
252            advance(lexer);
253        }
254    } else if (lexer->lookahead == '-') {
255        for (unsigned i = 0; i < 3; i++) {
256            if (lexer->lookahead != comment_end[i]) {
257                return false;
258            }
259            advance(lexer);
260        }
261    } else {
262        return false;
263    }
264
265    while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 &&
266           lexer->lookahead != 0x2029) {
267        advance(lexer);
268    }
269
270    lexer->result_symbol = HTML_COMMENT;
271    lexer->mark_end(lexer);
272
273    return true;
274}
275
276static bool scan_jsx_text(TSLexer *lexer) {
277    // saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and
278    // does not immediately follow a newline.
279    bool saw_text = false;
280    // at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but
281    // immediately follows a newline.
282    bool at_newline = false;
283
284    while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' &&
285           lexer->lookahead != '}' && lexer->lookahead != '&') {
286        bool is_wspace = iswspace(lexer->lookahead);
287        if (lexer->lookahead == '\n') {
288            at_newline = true;
289        } else {
290            // If at_newline is already true, and we see some whitespace, then it must stay true.
291            // Otherwise, it should be false.
292            //
293            // See the table below to determine the logic for computing `saw_text`.
294            //
295            // |------------------------------------|
296            // | at_newline | is_wspace | saw_text  |
297            // |------------|-----------|-----------|
298            // | false (0)  | false (0) | true  (1) |
299            // | false (0)  | true  (1) | true  (1) |
300            // | true  (1)  | false (0) | true  (1) |
301            // | true  (1)  | true  (1) | false (0) |
302            // |------------------------------------|
303
304            at_newline &= is_wspace;
305            if (!at_newline) {
306                saw_text = true;
307            }
308        }
309
310        advance(lexer);
311    }
312
313    lexer->result_symbol = JSX_TEXT;
314    return saw_text;
315}
316
317static inline bool external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
318    if (valid_symbols[TEMPLATE_CHARS]) {
319        if (valid_symbols[AUTOMATIC_SEMICOLON]) {
320            return false;
321        }
322        return scan_template_chars(lexer);
323    }
324
325    if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) {
326        return true;
327    }
328
329    if (valid_symbols[AUTOMATIC_SEMICOLON] || valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
330        bool scanned_comment = false;
331        bool ret = scan_automatic_semicolon(lexer, valid_symbols, &scanned_comment);
332        if (!ret && !scanned_comment && valid_symbols[TERNARY_QMARK] && lexer->lookahead == '?') {
333            return scan_ternary_qmark(lexer);
334        }
335        return ret;
336    }
337    if (valid_symbols[TERNARY_QMARK]) {
338        return scan_ternary_qmark(lexer);
339    }
340
341    if (valid_symbols[HTML_COMMENT] && !valid_symbols[LOGICAL_OR] && !valid_symbols[ESCAPE_SEQUENCE] &&
342        !valid_symbols[REGEX_PATTERN]) {
343        return scan_closing_comment(lexer);
344    }
345
346    return false;
347}