sascha
/
difftastic
mirror of https://github.com/Wilfred/difftastic/
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add experimental string format parsing

pull/70/head
Stephan Seitz 2021-04-18 17:52:13 +07:00
parent afe6feafce
commit 4ce4a62efe
5 changed files with 86302 additions and 71679 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
grammar.js
Unescape Escape View File

@ -35,10 +35,14 @@ const SYMBOL =
token(seq(SYMBOL_HEAD,
repeat(SYMBOL_BODY)));
function clSymbol(symbol){
function clSymbol(symbol) {
return seq(optional(seq('cl', ':')), symbol)
}
function srepeat(...args) {
return repeat(seq(...args))
}
module.exports = grammar(clojure, {
name: 'commonlisp',
@ -77,6 +81,52 @@ module.exports = grammar(clojure, {
repeat(choice(field('value', $._form), $._gap)),
field('close', ")")),
// https://en.wikipedia.org/wiki/Format_Common_Lisp)
format_prefix_parameters: _ => choice('v', 'V', '#'),
format_modifiers: _ => choice('@', '@:', ':', ':@'),
format_directive_type: $ => choice(
seq(optional(field('repetitions', choice($.num_lit))), choice('~', '%', '&', '|')),
/[cC]/,
/\^/,
'\n',
'\r',
/[pP]/,
/[iI]/,
/[wW]/,
/[sS]/,
/[aA]/,
'_',
/[()]/,
/[{}]/,
/[\[\]]/,
/[<>]/,
';',
seq(field('numberOfArgs', $.num_lit), '*'),
seq('/', $._form, '/', $._form, /[tT]/),
'?',
"Newline",
seq(repeat(choice($.num_lit, ',')), /[$rRbBdDgGxXeEoO]/),
),
format_specifier: $ =>
prec.left(seq(
'~',
optional($.format_prefix_parameters),
optional($.format_modifiers),
prec(5, $.format_directive_type),
)),
str_lit: $ =>
seq(
'"',
repeat(choice(
token.immediate(prec(1, /[^\\~"]+/)),
token.immediate(seq("\\\"")),
$.format_specifier
)),
optional('~'),
'"',
),
for_clause_word: _ => clSymbol(choice(
'in',
'across',
@ -188,43 +238,43 @@ module.exports = grammar(clojure, {
seq(SYMBOL),
_form: $ =>
seq(optional('#'),
choice(// atom-ish
$.num_lit,
$.fancy_literal,
$.kwd_lit,
$.str_lit,
$.char_lit,
$.nil_lit,
//$.bool_lit,
$.package_lit,
$.sym_lit,
// basic collection-ish
$.list_lit,
//$.map_lit,
$.vec_lit,
// dispatch reader macros
$.set_lit,
//$.anon_fn_lit,
//$.regex_lit,
$.read_cond_lit,
$.splicing_read_cond_lit,
//$.ns_map_lit,
$.var_quoting_lit,
$.sym_val_lit,
$.evaling_lit,
//$.tagged_or_ctor_lit,
// some other reader macros
$.derefing_lit,
$.quoting_lit,
$.syn_quoting_lit,
$.unquote_splicing_lit,
$.unquoting_lit,
$.include_reader_macro,
$.complex_num_lit,
".",
//seq($._gap, '.'),
)),
seq(optional('#'),
choice(// atom-ish
$.num_lit,
$.fancy_literal,
$.kwd_lit,
$.str_lit,
$.char_lit,
$.nil_lit,
//$.bool_lit,
$.package_lit,
$.sym_lit,
// basic collection-ish
$.list_lit,
//$.map_lit,
$.vec_lit,
// dispatch reader macros
$.set_lit,
//$.anon_fn_lit,
//$.regex_lit,
$.read_cond_lit,
$.splicing_read_cond_lit,
//$.ns_map_lit,
$.var_quoting_lit,
$.sym_val_lit,
$.evaling_lit,
//$.tagged_or_ctor_lit,
// some other reader macros
$.derefing_lit,
$.quoting_lit,
$.syn_quoting_lit,
$.unquote_splicing_lit,
$.unquoting_lit,
$.include_reader_macro,
$.complex_num_lit,
".",
//seq($._gap, '.'),
)),
include_reader_macro: $ =>
seq(repeat($._metadata_lit),

362
src/grammar.json
Unescape Escape View File

@ -471,50 +471,64 @@
}
},
"str_lit": {
"type": "TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "PATTERN",
"value": "[^\"\\\\]"
}
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\"
},
{
"type": "PATTERN",
"value": "."
},
{
"type": "REPEAT",
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\""
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PREC",
"value": 1,
"content": {
"type": "PATTERN",
"value": "[^\"\\\\]"
"value": "[^\\\\~\"]+"
}
}
]
}
},
{
"type": "STRING",
"value": "\""
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "\\\""
}
]
}
},
{
"type": "SYMBOL",
"name": "format_specifier"
}
]
}
]
}
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "~"
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "\""
}
]
},
"char_lit": {
"type": "SEQ",
@ -1762,6 +1776,276 @@
}
]
},
"format_prefix_parameters": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "v"
},
{
"type": "STRING",
"value": "V"
},
{
"type": "STRING",
"value": "#"
}
]
},
"format_modifiers": {
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "@"
},
{
"type": "STRING",
"value": "@:"
},
{
"type": "STRING",
"value": ":"
},
{
"type": "STRING",
"value": ":@"
}
]
},
"format_directive_type": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "FIELD",
"name": "repetitions",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "num_lit"
}
]
}
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "~"
},
{
"type": "STRING",
"value": "%"
},
{
"type": "STRING",
"value": "&"
},
{
"type": "STRING",
"value": "|"
}
]
}
]
},
{
"type": "PATTERN",
"value": "[cC]"
},
{
"type": "PATTERN",
"value": "\\^"
},
{
"type": "STRING",
"value": "\n"
},
{
"type": "STRING",
"value": "\r"
},
{
"type": "PATTERN",
"value": "[pP]"
},
{
"type": "PATTERN",
"value": "[iI]"
},
{
"type": "PATTERN",
"value": "[wW]"
},
{
"type": "PATTERN",
"value": "[sS]"
},
{
"type": "PATTERN",
"value": "[aA]"
},
{
"type": "STRING",
"value": "_"
},
{
"type": "PATTERN",
"value": "[()]"
},
{
"type": "PATTERN",
"value": "[{}]"
},
{
"type": "PATTERN",
"value": "[\\[\\]]"
},
{
"type": "PATTERN",
"value": "[<>]"
},
{
"type": "STRING",
"value": ";"
},
{
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "numberOfArgs",
"content": {
"type": "SYMBOL",
"name": "num_lit"
}
},
{
"type": "STRING",
"value": "*"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "/"
},
{
"type": "SYMBOL",
"name": "_form"
},
{
"type": "STRING",
"value": "/"
},
{
"type": "SYMBOL",
"name": "_form"
},
{
"type": "PATTERN",
"value": "[tT]"
}
]
},
{
"type": "STRING",
"value": "?"
},
{
"type": "STRING",
"value": "Newline"
},
{
"type": "SEQ",
"members": [
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "num_lit"
},
{
"type": "STRING",
"value": ","
}
]
}
},
{
"type": "PATTERN",
"value": "[$rRbBdDgGxXeEoO]"
}
]
}
]
},
"format_specifier": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "~"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "format_prefix_parameters"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "format_modifiers"
},
{
"type": "BLANK"
}
]
},
{
"type": "PREC",
"value": 5,
"content": {
"type": "SYMBOL",
"name": "format_directive_type"
}
}
]
}
},
"for_clause_word": {
"type": "SEQ",
"members": [

237
src/node-types.json
Unescape Escape View File

@ -1468,6 +1468,163 @@
"named": true,
"fields": {}
},
{
"type": "format_directive_type",
"named": true,
"fields": {
"numberOfArgs": {
"multiple": false,
"required": false,
"types": [
{
"type": "num_lit",
"named": true
}
]
},
"repetitions": {
"multiple": false,
"required": false,
"types": [
{
"type": "num_lit",
"named": true
}
]
}
},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "char_lit",
"named": true
},
{
"type": "complex_num_lit",
"named": true
},
{
"type": "derefing_lit",
"named": true
},
{
"type": "evaling_lit",
"named": true
},
{
"type": "fancy_literal",
"named": true
},
{
"type": "include_reader_macro",
"named": true
},
{
"type": "kwd_lit",
"named": true
},
{
"type": "list_lit",
"named": true
},
{
"type": "nil_lit",
"named": true
},
{
"type": "num_lit",
"named": true
},
{
"type": "package_lit",
"named": true
},
{
"type": "quoting_lit",
"named": true
},
{
"type": "read_cond_lit",
"named": true
},
{
"type": "set_lit",
"named": true
},
{
"type": "splicing_read_cond_lit",
"named": true
},
{
"type": "str_lit",
"named": true
},
{
"type": "sym_lit",
"named": true
},
{
"type": "sym_val_lit",
"named": true
},
{
"type": "syn_quoting_lit",
"named": true
},
{
"type": "unquote_splicing_lit",
"named": true
},
{
"type": "unquoting_lit",
"named": true
},
{
"type": "var_quoting_lit",
"named": true
},
{
"type": "vec_lit",
"named": true
}
]
}
},
{
"type": "format_modifiers",
"named": true,
"fields": {}
},
{
"type": "format_prefix_parameters",
"named": true,
"fields": {}
},
{
"type": "format_specifier",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "format_directive_type",
"named": true
},
{
"type": "format_modifiers",
"named": true
},
{
"type": "format_prefix_parameters",
"named": true
}
]
}
},
{
"type": "include_reader_macro",
"named": true,
@ -3253,7 +3410,17 @@
{
"type": "str_lit",
"named": true,
"fields": {}
"fields": {},
"children": {
"multiple": true,
"required": false,
"types": [
{
"type": "format_specifier",
"named": true
}
]
}
},
{
"type": "sym_lit",
@ -4293,6 +4460,18 @@
]
}
},
{
"type": "\n",
"named": false
},
{
"type": "\r",
"named": false
},
{
"type": "\"",
"named": false
},
{
"type": "#",
"named": false
@ -4349,6 +4528,14 @@
"type": "#c",
"named": false
},
{
"type": "%",
"named": false
},
{
"type": "&",
"named": false
},
{
"type": "'",
"named": false
@ -4361,6 +4548,10 @@
"type": ")",
"named": false
},
{
"type": "*",
"named": false
},
{
"type": ",",
"named": false
@ -4373,6 +4564,10 @@
"type": ".",
"named": false
},
{
"type": "/",
"named": false
},
{
"type": ":",
"named": false
@ -4381,22 +4576,50 @@
"type": "::",
"named": false
},
{
"type": ":@",
"named": false
},
{
"type": ";",
"named": false
},
{
"type": "=",
"named": false
},
{
"type": "?",
"named": false
},
{
"type": "@",
"named": false
},
{
"type": "@:",
"named": false
},
{
"type": "A",
"named": false
},
{
"type": "Newline",
"named": false
},
{
"type": "V",
"named": false
},
{
"type": "^",
"named": false
},
{
"type": "_",
"named": false
},
{
"type": "`",
"named": false
@ -4565,6 +4788,10 @@
"type": "using",
"named": false
},
{
"type": "v",
"named": false
},
{
"type": "when",
"named": false
@ -4581,8 +4808,16 @@
"type": "{",
"named": false
},
{
"type": "|",
"named": false
},
{
"type": "}",
"named": false
},
{
"type": "~",
"named": false
}
]

157067
src/parser.c
View File

File diff suppressed because it is too large Load Diff

189
test/corpus/basic.txt
Unescape Escape View File

@ -388,7 +388,13 @@ Loop Macro (finally)
(list_lit
(sym_lit)
(sym_lit)
(str_lit)
(str_lit
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type)))
(sym_lit)
(sym_lit))))
(loop_clause
@ -444,14 +450,26 @@ Loop Macro (do do do)
(list_lit
(sym_lit)
(sym_lit)
(str_lit)
(str_lit
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type)))
(sym_lit)
(sym_lit))))
(loop_clause
(list_lit
(sym_lit)
(sym_lit)
(str_lit)
(str_lit
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type))
(format_specifier
(format_directive_type)))
(sym_lit)
(sym_lit)))
(loop_clause
@ -795,169 +813,26 @@ Num Literals (TODO)
--------------------------------------------------------------------------------
(source
(list_lit
(ERROR
(sym_lit)
(sym_lit)
(list_lit)
(quoting_lit
(list_lit
(sym_lit)
(num_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(num_lit)
(sym_lit)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'S'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)))
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'S'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit))
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'S'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'S'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'F'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'F'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'F'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'F'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'D'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'D'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'D'))
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'D'))
(sym_lit)
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(num_lit)
(ERROR
(UNEXPECTED 'L'))
(sym_lit)
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(sym_lit)
(num_lit)
(ERROR
(UNEXPECTED 'L'))
(sym_lit)
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(sym_lit)
(num_lit)
(ERROR
(UNEXPECTED 'L'))
(sym_lit)
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR)
(complex_num_lit
(sym_lit)
(num_lit)
(ERROR
(UNEXPECTED 'L'))
(sym_lit)
(num_lit)
(MISSING "))
(num_lit)
(sym_lit)
(ERROR))
(sym_lit)
(num_lit)))