sascha
/
difftastic
mirror of https://github.com/Wilfred/difftastic/
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add punctuation explicitly to grammar

pull/261/head
Wilfred Hughes 2022-04-17 17:30:00 +07:00
parent 0509e440ae
commit 06dfec714e
6 changed files with 129790 additions and 103883 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
corpus/declarations.txt
Unescape Escape View File

@ -125,6 +125,7 @@ fn accumulate(self) -> Machine<{State::Accumulate}> {}
macro: (identifier)
(token_tree
(string_literal)
(non_special_punctuation)
(identifier))))))
(function_item
name: (identifier)
@ -1046,27 +1047,39 @@ foo(#[bar(some tokens are special in other contexts: $/';()*()+.)] x);
(call_expression
function: (identifier)
arguments: (arguments
(attribute_item (attr_item
(identifier)
arguments: (token_tree (identifier) (identifier))))
(attribute_item
(attr_item
(identifier)
arguments: (token_tree
(non_special_punctuation)
(identifier)
(identifier)
(non_special_punctuation))))
(identifier)
(identifier))))
(expression_statement
(call_expression
function: (identifier)
arguments: (arguments
(attribute_item (attr_item
(identifier)
arguments: (token_tree
(identifier)
(identifier)
(attribute_item
(attr_item
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(token_tree)
(token_tree))))
arguments: (token_tree
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(non_special_punctuation)
(non_special_punctuation)
(non_special_punctuation)
(token_tree)
(non_special_punctuation)
(token_tree)
(non_special_punctuation)
(non_special_punctuation))))
(identifier)))))
================================================================================
@ -1090,36 +1103,59 @@ pub enum Error {
(source_file
(line_comment)
(use_declaration
(scoped_identifier (identifier) (identifier)))
(scoped_identifier
(identifier)
(identifier)))
(attribute_item
(meta_item (identifier)
(meta_item
(identifier)
(meta_arguments
(meta_item (identifier))
(meta_item (identifier)))))
(meta_item
(identifier))
(meta_item
(identifier)))))
(enum_item
(visibility_modifier)
(type_identifier)
(enum_variant_list
(attribute_item (attr_item
(identifier)
(token_tree
(string_literal)
(attribute_item
(attr_item
(identifier)
(token_tree (integer_literal)))))
(enum_variant (identifier)
(ordered_field_declaration_list (type_identifier)))
(attribute_item (attr_item
(token_tree
(string_literal)
(non_special_punctuation)
(identifier)
(token_tree
(non_special_punctuation)
(integer_literal)))))
(enum_variant
(identifier)
(token_tree
(string_literal)
(identifier)
(identifier)
(ordered_field_declaration_list
(type_identifier)))
(attribute_item
(attr_item
(identifier)
(identifier))))
(enum_variant (identifier)
(token_tree
(string_literal)
(non_special_punctuation)
(non_special_punctuation)
(identifier)
(non_special_punctuation)
(identifier)
(non_special_punctuation)
(non_special_punctuation)
(identifier)
(non_special_punctuation)
(identifier))))
(enum_variant
(identifier)
(field_declaration_list
(field_declaration (field_identifier) (primitive_type))
(field_declaration (field_identifier) (type_identifier)))))))
(field_declaration
(field_identifier)
(primitive_type))
(field_declaration
(field_identifier)
(type_identifier)))))))
================================================================================
Attributes and Expressions

49
corpus/macros.txt
Unescape Escape View File

@ -64,34 +64,45 @@ a!($a $a:ident $($a);*);
(macro_invocation
(identifier)
(token_tree
(identifier))))
(non_special_punctuation)
(identifier)
(non_special_punctuation))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(non_special_punctuation)
(identifier)
(non_special_punctuation))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier))))
(non_special_punctuation)
(identifier)
(non_special_punctuation))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier)
(identifier))))
(non_special_punctuation)
(identifier)
(non_special_punctuation)
(non_special_punctuation))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(char_literal)
(non_special_punctuation)
(char_literal))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(char_literal)
(non_special_punctuation)
(char_literal))))
(macro_invocation
(identifier)
@ -114,24 +125,33 @@ a!($a $a:ident $($a);*);
(expression_statement
(macro_invocation
(identifier)
(token_tree (token_tree))))
(token_tree
(token_tree))))
(expression_statement
(macro_invocation
(identifier)
(token_tree (identifier))))
(token_tree
(identifier))))
(expression_statement
(macro_invocation
(identifier)
(token_tree (token_tree (token_tree (token_tree (identifier)))))))
(token_tree
(token_tree
(token_tree
(token_tree
(identifier)))))))
(expression_statement
(macro_invocation
(identifier)
(token_tree
(identifier)
(identifier)
(non_special_punctuation)
(identifier)
(token_tree
(identifier))))))
(identifier))
(non_special_punctuation)
(non_special_punctuation)))))
================================================================================
Macro invocation with comments
@ -193,38 +213,47 @@ macro_rules! zero_or_one {
left: (token_tree_pattern)
right: (token_tree
(identifier)
(non_special_punctuation)
(token_tree
(string_literal)))))
(string_literal))
(non_special_punctuation))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern)
right: (token_tree
(integer_literal)
(non_special_punctuation)
(integer_literal))))
(macro_definition
name: (identifier)
(macro_rule
left: (token_tree_pattern
(identifier)
(non_special_punctuation)
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier)))
right: (token_tree
(identifier)
(non_special_punctuation)
(token_tree
(string_literal)
(non_special_punctuation)
(metavariable))))
(macro_rule
left: (token_tree_pattern
(identifier)
(non_special_punctuation)
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier)))
right: (token_tree
(identifier)
(non_special_punctuation)
(token_tree
(string_literal)
(non_special_punctuation)
(metavariable)))))
(macro_definition
name: (identifier)
@ -234,6 +263,7 @@ macro_rules! zero_or_one {
(token_binding_pattern
name: (metavariable)
type: (fragment_specifier))
(non_special_punctuation)
(token_tree_pattern
(token_repetition_pattern
(token_binding_pattern
@ -243,6 +273,7 @@ macro_rules! zero_or_one {
(token_repetition
(token_repetition
(metavariable)
(non_special_punctuation)
(metavariable))))))
(macro_definition
name: (identifier)

10
grammar.js
Unescape Escape View File

@ -240,13 +240,21 @@ module.exports = grammar({
'$', '(', repeat($._tokens), ')', optional(/[^+*?]+/), choice('+', '*', '?')
),
non_special_punctuation: $ => choice(
// https://doc.rust-lang.org/reference/tokens.html#punctuation
"+", "-", "*", "/", "%", "^", "!", "&", "|", "&&", "||", "<<",
">>", "+=", "-=", "*=", "/=", "%=", "^=", "&=", "|=", "<<=",
">>=", "=", "==", "!=", ">", "<", ">=", "<=", "@", "_", ".",
"..", "...", "..=", ",", ";", ":", "::", "->", "=>", "#", "?",
),
// Matches non-delimiter tokens common to both macro invocations and
// definitions. This is everything except $ and metavariables (which begin
// with $).
_non_special_token: $ => choice(
$._literal, $.identifier, $.mutable_specifier, $.self, $.super, $.crate,
alias(choice(...primitive_types), $.primitive_type),
/[/_\-=->,;:::!=?.@*&#%^+<>|~]+/,
$.non_special_punctuation,
'\'',
'as', 'async', 'await', 'break', 'const', 'continue', 'default', 'enum', 'fn', 'for', 'if', 'impl',
'let', 'loop', 'match', 'mod', 'pub', 'return', 'static', 'struct', 'trait', 'type',

185
src/grammar.json vendored
Unescape Escape View File

@ -661,6 +661,187 @@
}
]
},
"non_special_punctuation": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "+"
},
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "*"
},
{
"type": "STRING",
"value": "/"
},
{
"type": "STRING",
"value": "%"
},
{
"type": "STRING",
"value": "^"
},
{
"type": "STRING",
"value": "!"
},
{
"type": "STRING",
"value": "&"
},
{
"type": "STRING",
"value": "|"
},
{
"type": "STRING",
"value": "&&"
},
{
"type": "STRING",
"value": "||"
},
{
"type": "STRING",
"value": "<<"
},
{
"type": "STRING",
"value": ">>"
},
{
"type": "STRING",
"value": "+="
},
{
"type": "STRING",
"value": "-="
},
{
"type": "STRING",
"value": "*="
},
{
"type": "STRING",
"value": "/="
},
{
"type": "STRING",
"value": "%="
},
{
"type": "STRING",
"value": "^="
},
{
"type": "STRING",
"value": "&="
},
{
"type": "STRING",
"value": "|="
},
{
"type": "STRING",
"value": "<<="
},
{
"type": "STRING",
"value": ">>="
},
{
"type": "STRING",
"value": "="
},
{
"type": "STRING",
"value": "=="
},
{
"type": "STRING",
"value": "!="
},
{
"type": "STRING",
"value": ">"
},
{
"type": "STRING",
"value": "<"
},
{
"type": "STRING",
"value": ">="
},
{
"type": "STRING",
"value": "<="
},
{
"type": "STRING",
"value": "@"
},
{
"type": "STRING",
"value": "_"
},
{
"type": "STRING",
"value": "."
},
{
"type": "STRING",
"value": ".."
},
{
"type": "STRING",
"value": "..."
},
{
"type": "STRING",
"value": "..="
},
{
"type": "STRING",
"value": ","
},
{
"type": "STRING",
"value": ";"
},
{
"type": "STRING",
"value": ":"
},
{
"type": "STRING",
"value": "::"
},
{
"type": "STRING",
"value": "->"
},
{
"type": "STRING",
"value": "=>"
},
{
"type": "STRING",
"value": "#"
},
{
"type": "STRING",
"value": "?"
}
]
},
"_non_special_token": {
"type": "CHOICE",
"members": [
@ -767,8 +948,8 @@
"value": "primitive_type"
},
{
"type": "PATTERN",
"value": "[/_\\-=->,;:::!=?.@*&#%^+<>|~]+"
"type": "SYMBOL",
"name": "non_special_punctuation"
},
{
"type": "STRING",

21
src/node-types.json vendored
Unescape Escape View File

@ -2890,6 +2890,11 @@
]
}
},
{
"type": "non_special_punctuation",
"named": true,
"fields": {}
},
{
"type": "optional_type_parameter",
"named": true,
@ -3753,6 +3758,10 @@
"type": "mutable_specifier",
"named": true
},
{
"type": "non_special_punctuation",
"named": true
},
{
"type": "primitive_type",
"named": true
@ -3804,6 +3813,10 @@
"type": "mutable_specifier",
"named": true
},
{
"type": "non_special_punctuation",
"named": true
},
{
"type": "primitive_type",
"named": true
@ -3859,6 +3872,10 @@
"type": "mutable_specifier",
"named": true
},
{
"type": "non_special_punctuation",
"named": true
},
{
"type": "primitive_type",
"named": true
@ -3910,6 +3927,10 @@
"type": "mutable_specifier",
"named": true
},
{
"type": "non_special_punctuation",
"named": true
},
{
"type": "primitive_type",
"named": true

233304
src/parser.c vendored
View File

File diff suppressed because it is too large Load Diff