1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
#include "tree_sitter/alloc.h"
#include "tree_sitter/parser.h"
#include <assert.h>
#include <string.h>
#include <wctype.h>
enum TokenType { RAW_STRING_DELIMITER, RAW_STRING_CONTENT };
/// The spec limits delimiters to 16 chars
#define MAX_DELIMITER_LENGTH 16
typedef struct {
uint8_t delimiter_length;
wchar_t delimiter[MAX_DELIMITER_LENGTH];
} Scanner;
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static inline void reset(Scanner *scanner) {
scanner->delimiter_length = 0;
memset(scanner->delimiter, 0, sizeof scanner->delimiter);
}
/// Scan the raw string delimiter in R"delimiter(content)delimiter"
static bool scan_raw_string_delimiter(Scanner *scanner, TSLexer *lexer) {
if (scanner->delimiter_length > 0) {
// Closing delimiter: must exactly match the opening delimiter.
// We already checked this when scanning content, but this is how we
// know when to stop. We can't stop at ", because R"""hello""" is valid.
for (int i = 0; i < scanner->delimiter_length; ++i) {
if (lexer->lookahead != scanner->delimiter[i]) {
return false;
}
advance(lexer);
}
reset(scanner);
return true;
}
// Opening delimiter: record the d-char-sequence up to (.
// d-char is any basic character except parens, backslashes, and spaces.
for (;;) {
if (scanner->delimiter_length >= MAX_DELIMITER_LENGTH || lexer->eof(lexer) || lexer->lookahead == '\\' ||
iswspace(lexer->lookahead)) {
return false;
}
if (lexer->lookahead == '(') {
// Rather than create a token for an empty delimiter, we fail and
// let the grammar fall back to a delimiter-less rule.
return scanner->delimiter_length > 0;
}
scanner->delimiter[scanner->delimiter_length++] = lexer->lookahead;
advance(lexer);
}
}
/// Scan the raw string content in R"delimiter(content)delimiter"
static bool scan_raw_string_content(Scanner *scanner, TSLexer *lexer) {
// The progress made through the delimiter since the last ')'.
// The delimiter may not contain ')' so a single counter suffices.
for (int delimiter_index = -1;;) {
// If we hit EOF, consider the content to terminate there.
// This forms an incomplete raw_string_literal, and models the code
// well.
if (lexer->eof(lexer)) {
lexer->mark_end(lexer);
return true;
}
if (delimiter_index >= 0) {
if (delimiter_index == scanner->delimiter_length) {
if (lexer->lookahead == '"') {
return true;
}
delimiter_index = -1;
} else {
if (lexer->lookahead == scanner->delimiter[delimiter_index]) {
delimiter_index += 1;
} else {
delimiter_index = -1;
}
}
}
if (delimiter_index == -1 && lexer->lookahead == ')') {
// The content doesn't include the )delimiter" part.
// We must still scan through it, but exclude it from the token.
lexer->mark_end(lexer);
delimiter_index = 0;
}
advance(lexer);
}
}
void *tree_sitter_cuda_external_scanner_create() {
Scanner *scanner = (Scanner *)ts_calloc(1, sizeof(Scanner));
memset(scanner, 0, sizeof(Scanner));
return scanner;
}
bool tree_sitter_cuda_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
if (valid_symbols[RAW_STRING_DELIMITER] && valid_symbols[RAW_STRING_CONTENT]) {
// we're in error recovery
return false;
}
// No skipping leading whitespace: raw-string grammar is space-sensitive.
if (valid_symbols[RAW_STRING_DELIMITER]) {
lexer->result_symbol = RAW_STRING_DELIMITER;
return scan_raw_string_delimiter(scanner, lexer);
}
if (valid_symbols[RAW_STRING_CONTENT]) {
lexer->result_symbol = RAW_STRING_CONTENT;
return scan_raw_string_content(scanner, lexer);
}
return false;
}
unsigned tree_sitter_cuda_external_scanner_serialize(void *payload, char *buffer) {
static_assert(MAX_DELIMITER_LENGTH * sizeof(wchar_t) < TREE_SITTER_SERIALIZATION_BUFFER_SIZE,
"Serialized delimiter is too long!");
Scanner *scanner = (Scanner *)payload;
size_t size = scanner->delimiter_length * sizeof(wchar_t);
memcpy(buffer, scanner->delimiter, size);
return (unsigned)size;
}
void tree_sitter_cuda_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
assert(length % sizeof(wchar_t) == 0 && "Can't decode serialized delimiter!");
Scanner *scanner = (Scanner *)payload;
scanner->delimiter_length = length / sizeof(wchar_t);
if (length > 0) {
memcpy(&scanner->delimiter[0], buffer, length);
}
}
void tree_sitter_cuda_external_scanner_destroy(void *payload) {
Scanner *scanner = (Scanner *)payload;
ts_free(scanner);
}
|
