|
|
|
|
@ -1,7 +1,5 @@
|
|
|
|
|
#include <tree_sitter/parser.h>
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
|
|
// See references in grammar.externals
|
|
|
|
|
enum TokenType {
|
|
|
|
|
QUOTED_CONTENT_I_SINGLE,
|
|
|
|
|
@ -36,37 +34,31 @@ enum TokenType {
|
|
|
|
|
QUOTED_ATOM_START
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
void advance(TSLexer* lexer) {
|
|
|
|
|
lexer->advance(lexer, false);
|
|
|
|
|
}
|
|
|
|
|
static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
|
|
|
|
|
|
|
|
|
|
void skip(TSLexer *lexer) {
|
|
|
|
|
lexer->advance(lexer, true);
|
|
|
|
|
}
|
|
|
|
|
static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
|
|
|
|
|
|
|
|
|
|
// Note: some checks require several lexer steps of lookahead
|
|
|
|
|
// and alter its state, for these we use names check_*
|
|
|
|
|
|
|
|
|
|
bool is_whitespace(int32_t c) {
|
|
|
|
|
static inline bool is_whitespace(int32_t c) {
|
|
|
|
|
return c == ' ' || c == '\t' || c == '\n' || c == '\r';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool is_inline_whitespace(int32_t c) {
|
|
|
|
|
static inline bool is_inline_whitespace(int32_t c) {
|
|
|
|
|
return c == ' ' || c == '\t';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool is_newline(int32_t c) {
|
|
|
|
|
static inline bool is_newline(int32_t c) {
|
|
|
|
|
// Note: this implies \r\n is treated as two line breaks,
|
|
|
|
|
// but in our case it's fine, since multiple line breaks
|
|
|
|
|
// make no difference
|
|
|
|
|
return c == '\n' || c == '\r';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool is_digit(int32_t c) {
|
|
|
|
|
return '0' <= c && c <= '9';
|
|
|
|
|
}
|
|
|
|
|
static inline bool is_digit(int32_t c) { return '0' <= c && c <= '9'; }
|
|
|
|
|
|
|
|
|
|
bool check_keyword_end(TSLexer* lexer) {
|
|
|
|
|
static inline bool check_keyword_end(TSLexer *lexer) {
|
|
|
|
|
if (lexer->lookahead == ':') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return is_whitespace(lexer->lookahead);
|
|
|
|
|
@ -74,7 +66,7 @@ bool check_keyword_end(TSLexer* lexer) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool check_operator_end(TSLexer* lexer) {
|
|
|
|
|
static bool check_operator_end(TSLexer *lexer) {
|
|
|
|
|
// Keyword
|
|
|
|
|
if (lexer->lookahead == ':') {
|
|
|
|
|
return !check_keyword_end(lexer);
|
|
|
|
|
@ -97,21 +89,22 @@ bool check_operator_end(TSLexer* lexer) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const char token_terminators[] = {
|
|
|
|
|
// Operator starts
|
|
|
|
|
'@', '.', '+', '-', '^', '-', '*', '/', '<', '>', '|', '~', '=', '&', '\\', '%',
|
|
|
|
|
// Delimiters
|
|
|
|
|
'{', '}', '[', ']', '(', ')', '"', '\'',
|
|
|
|
|
// Separators
|
|
|
|
|
',', ';',
|
|
|
|
|
// Comment
|
|
|
|
|
'#'
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const uint8_t token_terminators_length = sizeof(token_terminators) / sizeof(char);
|
|
|
|
|
// Operator starts
|
|
|
|
|
'@', '.', '+', '-', '^', '-', '*', '/', '<', '>', '|', '~', '=', '&', '\\',
|
|
|
|
|
'%',
|
|
|
|
|
// Delimiters
|
|
|
|
|
'{', '}', '[', ']', '(', ')', '"', '\'',
|
|
|
|
|
// Separators
|
|
|
|
|
',', ';',
|
|
|
|
|
// Comment
|
|
|
|
|
'#'};
|
|
|
|
|
|
|
|
|
|
const uint8_t token_terminators_length =
|
|
|
|
|
sizeof(token_terminators) / sizeof(char);
|
|
|
|
|
|
|
|
|
|
// Note: this is a heuristic as we only use this to distinguish word
|
|
|
|
|
// operators and we don't want to include complex Unicode ranges
|
|
|
|
|
bool is_token_end(int32_t c) {
|
|
|
|
|
static inline bool is_token_end(int32_t c) {
|
|
|
|
|
for (uint8_t i = 0; i < token_terminators_length; i++) {
|
|
|
|
|
if (c == token_terminators[i]) {
|
|
|
|
|
return true;
|
|
|
|
|
@ -121,44 +114,46 @@ bool is_token_end(int32_t c) {
|
|
|
|
|
return is_whitespace(c);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct QuotedContentInfo {
|
|
|
|
|
const TokenType token_type;
|
|
|
|
|
typedef struct {
|
|
|
|
|
const enum TokenType token_type;
|
|
|
|
|
const bool supports_interpol;
|
|
|
|
|
const int32_t end_delimiter;
|
|
|
|
|
const uint8_t delimiter_length;
|
|
|
|
|
};
|
|
|
|
|
} QuotedContentInfo;
|
|
|
|
|
|
|
|
|
|
const QuotedContentInfo quoted_content_infos[] = {
|
|
|
|
|
{ QUOTED_CONTENT_I_SINGLE, true, '\'', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_DOUBLE, true, '"', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_HEREDOC_SINGLE, true, '\'', 3 },
|
|
|
|
|
{ QUOTED_CONTENT_I_HEREDOC_DOUBLE, true, '"', 3 },
|
|
|
|
|
{ QUOTED_CONTENT_I_PARENTHESIS, true, ')', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_CURLY, true, '}', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_SQUARE, true, ']', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_ANGLE, true, '>', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_BAR, true, '|', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_I_SLASH, true, '/', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_SINGLE, false, '\'', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_DOUBLE, false, '"', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_HEREDOC_SINGLE, false, '\'', 3 },
|
|
|
|
|
{ QUOTED_CONTENT_HEREDOC_DOUBLE, false, '"', 3 },
|
|
|
|
|
{ QUOTED_CONTENT_PARENTHESIS, false, ')', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_CURLY, false, '}', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_SQUARE, false, ']', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_ANGLE, false, '>', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_BAR, false, '|', 1 },
|
|
|
|
|
{ QUOTED_CONTENT_SLASH, false, '/', 1 },
|
|
|
|
|
{QUOTED_CONTENT_I_SINGLE, true, '\'', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_DOUBLE, true, '"', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_HEREDOC_SINGLE, true, '\'', 3},
|
|
|
|
|
{QUOTED_CONTENT_I_HEREDOC_DOUBLE, true, '"', 3},
|
|
|
|
|
{QUOTED_CONTENT_I_PARENTHESIS, true, ')', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_CURLY, true, '}', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_SQUARE, true, ']', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_ANGLE, true, '>', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_BAR, true, '|', 1},
|
|
|
|
|
{QUOTED_CONTENT_I_SLASH, true, '/', 1},
|
|
|
|
|
{QUOTED_CONTENT_SINGLE, false, '\'', 1},
|
|
|
|
|
{QUOTED_CONTENT_DOUBLE, false, '"', 1},
|
|
|
|
|
{QUOTED_CONTENT_HEREDOC_SINGLE, false, '\'', 3},
|
|
|
|
|
{QUOTED_CONTENT_HEREDOC_DOUBLE, false, '"', 3},
|
|
|
|
|
{QUOTED_CONTENT_PARENTHESIS, false, ')', 1},
|
|
|
|
|
{QUOTED_CONTENT_CURLY, false, '}', 1},
|
|
|
|
|
{QUOTED_CONTENT_SQUARE, false, ']', 1},
|
|
|
|
|
{QUOTED_CONTENT_ANGLE, false, '>', 1},
|
|
|
|
|
{QUOTED_CONTENT_BAR, false, '|', 1},
|
|
|
|
|
{QUOTED_CONTENT_SLASH, false, '/', 1},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const uint8_t quoted_content_infos_length = sizeof(quoted_content_infos) / sizeof(QuotedContentInfo);
|
|
|
|
|
const uint8_t quoted_content_infos_length =
|
|
|
|
|
sizeof(quoted_content_infos) / sizeof(QuotedContentInfo);
|
|
|
|
|
|
|
|
|
|
int8_t find_quoted_token_info(const bool* valid_symbols) {
|
|
|
|
|
static inline int8_t find_quoted_token_info(const bool *valid_symbols) {
|
|
|
|
|
// Quoted tokens are mutually exclusive and only one should be valid
|
|
|
|
|
// at a time. If multiple are valid it means we parse an arbitrary
|
|
|
|
|
// code outside quotes, in which case we don't want to tokenize it as
|
|
|
|
|
// quoted content.
|
|
|
|
|
if (valid_symbols[QUOTED_CONTENT_I_SINGLE] && valid_symbols[QUOTED_CONTENT_I_DOUBLE]) {
|
|
|
|
|
if (valid_symbols[QUOTED_CONTENT_I_SINGLE] &&
|
|
|
|
|
valid_symbols[QUOTED_CONTENT_I_DOUBLE]) {
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -171,10 +166,10 @@ int8_t find_quoted_token_info(const bool* valid_symbols) {
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
|
|
|
|
|
lexer->result_symbol = info.token_type;
|
|
|
|
|
static bool scan_quoted_content(TSLexer *lexer, const QuotedContentInfo *info) {
|
|
|
|
|
lexer->result_symbol = info->token_type;
|
|
|
|
|
|
|
|
|
|
bool is_heredoc = (info.delimiter_length == 3);
|
|
|
|
|
bool is_heredoc = (info->delimiter_length == 3);
|
|
|
|
|
|
|
|
|
|
for (bool has_content = false; true; has_content = true) {
|
|
|
|
|
bool newline = false;
|
|
|
|
|
@ -192,33 +187,35 @@ bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
|
|
|
|
|
|
|
|
|
|
lexer->mark_end(lexer);
|
|
|
|
|
|
|
|
|
|
if (lexer->lookahead == info.end_delimiter) {
|
|
|
|
|
if (lexer->lookahead == info->end_delimiter) {
|
|
|
|
|
uint8_t length = 1;
|
|
|
|
|
|
|
|
|
|
while (length < info.delimiter_length) {
|
|
|
|
|
while (length < info->delimiter_length) {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == info.end_delimiter) {
|
|
|
|
|
if (lexer->lookahead == info->end_delimiter) {
|
|
|
|
|
length++;
|
|
|
|
|
} else {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (length == info.delimiter_length && (!is_heredoc || newline)) {
|
|
|
|
|
if (length == info->delimiter_length && (!is_heredoc || newline)) {
|
|
|
|
|
return has_content;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (lexer->lookahead == '#') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (info.supports_interpol && lexer->lookahead == '{') {
|
|
|
|
|
if (info->supports_interpol && lexer->lookahead == '{') {
|
|
|
|
|
return has_content;
|
|
|
|
|
}
|
|
|
|
|
} else if (lexer->lookahead == '\\') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (is_heredoc && lexer->lookahead == '\n') {
|
|
|
|
|
// We need to know about the newline to correctly recognise
|
|
|
|
|
// heredoc end delimiter, so we intentionally ignore escaping
|
|
|
|
|
} else if (info.supports_interpol || lexer->lookahead == info.end_delimiter) {
|
|
|
|
|
// heredoc end delimiter, so we intentionally ignore
|
|
|
|
|
// escaping
|
|
|
|
|
} else if (info->supports_interpol ||
|
|
|
|
|
lexer->lookahead == info->end_delimiter) {
|
|
|
|
|
return has_content;
|
|
|
|
|
}
|
|
|
|
|
} else if (lexer->lookahead == '\0') {
|
|
|
|
|
@ -235,7 +232,7 @@ bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
static bool scan_newline(TSLexer *lexer, const bool *valid_symbols) {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
|
|
|
|
|
while (is_whitespace(lexer->lookahead)) {
|
|
|
|
|
@ -261,7 +258,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (valid_symbols[NEWLINE_BEFORE_BINARY_OPERATOR] ) {
|
|
|
|
|
if (valid_symbols[NEWLINE_BEFORE_BINARY_OPERATOR]) {
|
|
|
|
|
lexer->result_symbol = NEWLINE_BEFORE_BINARY_OPERATOR;
|
|
|
|
|
|
|
|
|
|
// &&, &&&
|
|
|
|
|
@ -276,7 +273,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// =, ==, ===, =~, =>
|
|
|
|
|
// =, ==, ===, =~, =>
|
|
|
|
|
} else if (lexer->lookahead == '=') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '=') {
|
|
|
|
|
@ -296,16 +293,17 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// ::
|
|
|
|
|
// ::
|
|
|
|
|
} else if (lexer->lookahead == ':') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == ':') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
// Ignore ::: atom
|
|
|
|
|
if (lexer->lookahead == ':') return false;
|
|
|
|
|
if (lexer->lookahead == ':')
|
|
|
|
|
return false;
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// ++, +++
|
|
|
|
|
// ++, +++
|
|
|
|
|
} else if (lexer->lookahead == '+') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '+') {
|
|
|
|
|
@ -317,7 +315,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// --, ---, ->
|
|
|
|
|
// --, ---, ->
|
|
|
|
|
} else if (lexer->lookahead == '-') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '-') {
|
|
|
|
|
@ -332,11 +330,10 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// <, <=, <-, <>, <~, <~>, <|>, <<<, <<~
|
|
|
|
|
// <, <=, <-, <>, <~, <~>, <|>, <<<, <<~
|
|
|
|
|
} else if (lexer->lookahead == '<') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '=' ||
|
|
|
|
|
lexer->lookahead == '-' ||
|
|
|
|
|
if (lexer->lookahead == '=' || lexer->lookahead == '-' ||
|
|
|
|
|
lexer->lookahead == '>') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
@ -356,15 +353,14 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
}
|
|
|
|
|
} else if (lexer->lookahead == '<') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '<' ||
|
|
|
|
|
lexer->lookahead == '~') {
|
|
|
|
|
if (lexer->lookahead == '<' || lexer->lookahead == '~') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// >, >=, >>>
|
|
|
|
|
// >, >=, >>>
|
|
|
|
|
} else if (lexer->lookahead == '>') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '=') {
|
|
|
|
|
@ -379,7 +375,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// ^^^
|
|
|
|
|
// ^^^
|
|
|
|
|
} else if (lexer->lookahead == '^') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '^') {
|
|
|
|
|
@ -389,7 +385,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// !=, !==
|
|
|
|
|
// !=, !==
|
|
|
|
|
} else if (lexer->lookahead == '!') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '=') {
|
|
|
|
|
@ -401,7 +397,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// ~>, ~>>
|
|
|
|
|
// ~>, ~>>
|
|
|
|
|
} else if (lexer->lookahead == '~') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '>') {
|
|
|
|
|
@ -413,7 +409,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// |, ||, |||, |>
|
|
|
|
|
// |, ||, |||, |>
|
|
|
|
|
} else if (lexer->lookahead == '|') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '|') {
|
|
|
|
|
@ -430,7 +426,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// *, **
|
|
|
|
|
// *, **
|
|
|
|
|
} else if (lexer->lookahead == '*') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '*') {
|
|
|
|
|
@ -439,7 +435,7 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// / //
|
|
|
|
|
// / //
|
|
|
|
|
} else if (lexer->lookahead == '/') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '/') {
|
|
|
|
|
@ -448,18 +444,19 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// ., ..
|
|
|
|
|
// ., ..
|
|
|
|
|
} else if (lexer->lookahead == '.') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '.') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
// Ignore ... identifier
|
|
|
|
|
if (lexer->lookahead == '.') return false;
|
|
|
|
|
if (lexer->lookahead == '.')
|
|
|
|
|
return false;
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
} else {
|
|
|
|
|
return check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// double slash
|
|
|
|
|
// double slash
|
|
|
|
|
} else if (lexer->lookahead == '\\') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '\\') {
|
|
|
|
|
@ -487,21 +484,21 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return is_token_end(lexer->lookahead) && check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// or
|
|
|
|
|
// or
|
|
|
|
|
} else if (lexer->lookahead == 'o') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == 'r') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return is_token_end(lexer->lookahead) && check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// in
|
|
|
|
|
// in
|
|
|
|
|
} else if (lexer->lookahead == 'i') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == 'n') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return is_token_end(lexer->lookahead) && check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
// not in
|
|
|
|
|
// not in
|
|
|
|
|
} else if (lexer->lookahead == 'n') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == 'o') {
|
|
|
|
|
@ -515,7 +512,8 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == 'n') {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
return is_token_end(lexer->lookahead) && check_operator_end(lexer);
|
|
|
|
|
return is_token_end(lexer->lookahead) &&
|
|
|
|
|
check_operator_end(lexer);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@ -526,14 +524,15 @@ bool scan_newline(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
static bool scan(TSLexer *lexer, const bool *valid_symbols) {
|
|
|
|
|
int8_t quoted_content_info_idx = find_quoted_token_info(valid_symbols);
|
|
|
|
|
|
|
|
|
|
// Quoted content, which matches any character except for close
|
|
|
|
|
// delimiters, escapes and interpolations
|
|
|
|
|
if (quoted_content_info_idx != -1) {
|
|
|
|
|
const QuotedContentInfo& info = quoted_content_infos[quoted_content_info_idx];
|
|
|
|
|
return scan_quoted_content(lexer, info);
|
|
|
|
|
const QuotedContentInfo info =
|
|
|
|
|
quoted_content_infos[quoted_content_info_idx];
|
|
|
|
|
return scan_quoted_content(lexer, &info);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool skipped_whitespace = false;
|
|
|
|
|
@ -544,10 +543,10 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Newline, which is either tokenized as a special newline or ignored
|
|
|
|
|
if (is_newline(lexer->lookahead) && (
|
|
|
|
|
valid_symbols[NEWLINE_BEFORE_DO] ||
|
|
|
|
|
valid_symbols[NEWLINE_BEFORE_BINARY_OPERATOR] ||
|
|
|
|
|
valid_symbols[NEWLINE_BEFORE_COMMENT])) {
|
|
|
|
|
if (is_newline(lexer->lookahead) &&
|
|
|
|
|
(valid_symbols[NEWLINE_BEFORE_DO] ||
|
|
|
|
|
valid_symbols[NEWLINE_BEFORE_BINARY_OPERATOR] ||
|
|
|
|
|
valid_symbols[NEWLINE_BEFORE_COMMENT])) {
|
|
|
|
|
return scan_newline(lexer, valid_symbols);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -556,7 +555,8 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
if (skipped_whitespace && valid_symbols[BEFORE_UNARY_OPERATOR]) {
|
|
|
|
|
lexer->mark_end(lexer);
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '+' || lexer->lookahead == ':' || lexer->lookahead == '/') {
|
|
|
|
|
if (lexer->lookahead == '+' || lexer->lookahead == ':' ||
|
|
|
|
|
lexer->lookahead == '/') {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
if (is_whitespace(lexer->lookahead)) {
|
|
|
|
|
@ -565,13 +565,14 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
lexer->result_symbol = BEFORE_UNARY_OPERATOR;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
// before unary -
|
|
|
|
|
// before unary -
|
|
|
|
|
} else if (lexer->lookahead == '-') {
|
|
|
|
|
if (skipped_whitespace && valid_symbols[BEFORE_UNARY_OPERATOR]) {
|
|
|
|
|
lexer->mark_end(lexer);
|
|
|
|
|
lexer->result_symbol = BEFORE_UNARY_OPERATOR;
|
|
|
|
|
advance(lexer);
|
|
|
|
|
if (lexer->lookahead == '-' || lexer->lookahead == '>' || lexer->lookahead == ':' || lexer->lookahead == '/') {
|
|
|
|
|
if (lexer->lookahead == '-' || lexer->lookahead == '>' ||
|
|
|
|
|
lexer->lookahead == ':' || lexer->lookahead == '/') {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
if (is_whitespace(lexer->lookahead)) {
|
|
|
|
|
@ -579,7 +580,7 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
// not in
|
|
|
|
|
// not in
|
|
|
|
|
} else if (lexer->lookahead == 'n') {
|
|
|
|
|
if (valid_symbols[NOT_IN]) {
|
|
|
|
|
lexer->result_symbol = NOT_IN;
|
|
|
|
|
@ -601,7 +602,7 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// quoted atom start
|
|
|
|
|
// quoted atom start
|
|
|
|
|
} else if (lexer->lookahead == ':') {
|
|
|
|
|
if (valid_symbols[QUOTED_ATOM_START]) {
|
|
|
|
|
advance(lexer);
|
|
|
|
|
@ -616,25 +617,20 @@ bool scan(TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Expose the API expected by tree-sitter
|
|
|
|
|
void *tree_sitter_elixir_external_scanner_create() { return NULL; }
|
|
|
|
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
void* tree_sitter_elixir_external_scanner_create() {
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool tree_sitter_elixir_external_scanner_scan(void* payload, TSLexer* lexer, const bool* valid_symbols) {
|
|
|
|
|
return scan(lexer, valid_symbols);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned tree_sitter_elixir_external_scanner_serialize(void* payload, char* buffer) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void tree_sitter_elixir_external_scanner_deserialize(void* payload, const char* buffer, unsigned length) {}
|
|
|
|
|
|
|
|
|
|
void tree_sitter_elixir_external_scanner_destroy(void* payload) {}
|
|
|
|
|
bool tree_sitter_elixir_external_scanner_scan(void *payload, TSLexer *lexer,
|
|
|
|
|
const bool *valid_symbols) {
|
|
|
|
|
return scan(lexer, valid_symbols);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// end anonymous namespace
|
|
|
|
|
unsigned tree_sitter_elixir_external_scanner_serialize(void *payload,
|
|
|
|
|
char *buffer) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void tree_sitter_elixir_external_scanner_deserialize(void *payload,
|
|
|
|
|
const char *buffer,
|
|
|
|
|
unsigned length) {}
|
|
|
|
|
|
|
|
|
|
void tree_sitter_elixir_external_scanner_destroy(void *payload) {}
|
Wilfred Hughes