Merge pull request #8 from theHamsta/fix-replacement-character

Fix replacement character

.github/workflows/ci.yml

@@ -23,7 +23,12 @@ jobs:
           if (( $(node_modules/tree-sitter-cli/tree-sitter parse test/Petalisp/**/*.lisp -q | wc -l) > 2 )); then # There are 2 known failures (strings that are not format strings but use ~X syntax)
             exit 1 
           else
-            echo "Successfully parsed Petalisp"
+            echo "Successfully parsed Petalisp."
+          fi
+          if (( $(node_modules/tree-sitter-cli/tree-sitter parse test/sly/**/*.lisp -q | wc -l) > 4 )); then # There are 2 known failures (strings that are not format strings but use ~X syntax)
+            exit 1 
+          else
+            echo "Successfully parsed Sly"
           fi
       - name: Run tests
         run: npm test

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "test/Petalisp"]
 	path = test/Petalisp
 	url = https://github.com/marcoheisig/Petalisp.git
+[submodule "test/sly"]
+	path = test/sly
+	url = https://github.com/joaotavora/sly.git

README.md

@@ -13,3 +13,5 @@ Macros with special respresentation in syntax tree (when written with lowercase
 
 - defun and friends (e.g. defmethod)
 - loop macro
+
+This grammar is used in https://github.com/Wilfred/difftastic to generate syntax-ware diffs for Common Lisp.

grammar.js

@@ -3,7 +3,7 @@
  * Copyright (C) 2021 Stephan Seitz <stephan.seitz@fau.de>
  * Adapted from tree-sitter-clojure
  *
- * Distributed under terms of the GPLv3 license.
+ * Distributed under terms of the MIT license.
  */
 
 const clojure = require("tree-sitter-clojure/grammar");
@@ -114,12 +114,28 @@ function clSymbol(symbol) {
     return seq(optional(seq('cl', ':')), symbol)
 }
 
+function loopSymbol(symbol) {
+    return seq(optional(seq(optional('cl'), ':')), symbol)
+}
+
+function optSeq(...args) {
+    return optional(seq(...args))
+}
+
 
 module.exports = grammar(clojure, {
     name: 'commonlisp',
 
     extras: ($, original) => [...original, $.block_comment],
-    conflicts: ($, original) => [...original, [$.for_clause], [$.accumulation_clause], [$.loop_macro, $.defun_keyword, $.package_lit]],
+    conflicts: ($,
+        original) => [...original,
+        [$.for_clause_word, $.package_lit],
+        [$.with_clause, $.package_lit],
+        [$.with_clause],
+        [$.for_clause],
+        [$.accumulation_clause],
+        [$.loop_macro, $.defun_keyword, $.package_lit]],
+
 
     rules: {
         block_comment: _ => token(seq('#|', repeat(choice(/[^|]/, /\|[^#]/)), '|#')),
@@ -193,14 +209,14 @@ module.exports = grammar(clojure, {
                 '"',
                 repeat(choice(
                     token.immediate(prec(1, /[^\\~"]+/)),
-                    token.immediate(seq("\\\"")),
+                    token.immediate(seq(/\\./)),
                     $.format_specifier,
                 )),
                 optional('~'),
                 '"',
             ),
 
-        for_clause_word: _ => clSymbol(choice(
+        for_clause_word: _ => loopSymbol(choice(
             'in',
             'across',
             'being',
@@ -211,6 +227,7 @@ module.exports = grammar(clojure, {
             'from',
             'to',
             'upto',
+            'upfrom',
             'downto',
             'downfrom',
             'on',
@@ -221,18 +238,16 @@ module.exports = grammar(clojure, {
 
         _for_part: $ => seq(repeat($._gap), $.for_clause_word, repeat($._gap), $._form),
 
-        accumulation_verb: _ => clSymbol(/((collect|append|nconc|count|maximize|minimize)(ing)?|sum(ming)?)/),
-
-        for_clause: $ => choice(seq(choice(clSymbol('for'), clSymbol('and'), clSymbol('as')), repeat($._gap), field('variable', $._form), optional(field('type', seq(repeat($._gap), $._form))),
-            repeat1($._for_part)), clSymbol('and')),
-
-        with_clause: $ => prec.left(seq(clSymbol('with'), repeat($._gap), $._form, optional(field('type', seq(repeat($._gap), $._form))), repeat($._gap), clSymbol("="), repeat($._gap), $._form)),
-        do_clause: $ => prec.left(seq(clSymbol('do'), repeat1(prec.left(seq(repeat($._gap), $._form, repeat($._gap)))))),
-        while_clause: $ => prec.left(seq(choice(clSymbol('while'), clSymbol('until')), repeat($._gap), $._form)),
-        repeat_clause: $ => prec.left(seq(clSymbol('repeat'), repeat($._gap), $._form)),
-        condition_clause: $ => prec.left(choice(seq(choice(clSymbol('when'), clSymbol('if'), clSymbol('unless'), clSymbol('always'), clSymbol('thereis'), clSymbol('never')), repeat($._gap), $._form), clSymbol("else"))),
-        accumulation_clause: $ => seq($.accumulation_verb, repeat($._gap), $._form, optional(seq(repeat($._gap), clSymbol('into'), repeat($._gap), $._form))),
-        termination_clause: $ => prec.left(seq(choice(clSymbol('finally'), clSymbol('return'), clSymbol('initially')), repeat($._gap), $._form)),
+        accumulation_verb: _ => loopSymbol(/((collect|append|nconc|count|maximize|minimize)(ing)?|sum(ming)?)/),
+        for_clause: $ => choice(seq(choice(loopSymbol('for'), loopSymbol('and'), loopSymbol('as')), repeat($._gap), field('variable', $._form), optional(field('type', seq(repeat($._gap), $._form))),
+            repeat1($._for_part)), loopSymbol('and')),
+        with_clause: $ => seq(loopSymbol('with'), repeat($._gap), choice($._form, seq($._form, repeat($._gap), field('type', $._form))), repeat($._gap), optSeq(loopSymbol("="), repeat($._gap)), optSeq($._form, repeat($._gap))),
+        do_clause: $ => prec.left(seq(loopSymbol('do'), repeat1(prec.left(seq(repeat($._gap), $._form, repeat($._gap)))))),
+        while_clause: $ => prec.left(seq(choice(loopSymbol('while'), loopSymbol('until')), repeat($._gap), $._form)),
+        repeat_clause: $ => prec.left(seq(loopSymbol('repeat'), repeat($._gap), $._form)),
+        condition_clause: $ => prec.left(choice(seq(choice(loopSymbol('when'), loopSymbol('if'), loopSymbol('unless'), loopSymbol('always'), loopSymbol('thereis'), loopSymbol('never')), repeat($._gap), $._form), loopSymbol("else"))),
+        accumulation_clause: $ => seq($.accumulation_verb, repeat($._gap), $._form, optional(seq(repeat($._gap), loopSymbol('into'), repeat($._gap), $._form))),
+        termination_clause: $ => prec.left(seq(choice(loopSymbol('finally'), loopSymbol('return'), loopSymbol('initially')), repeat($._gap), $._form)),
 
 
         loop_clause: $ =>
@@ -260,7 +275,11 @@ module.exports = grammar(clojure, {
         defun_keyword: _ => prec(10, clSymbol(choice('defun', 'defmacro', 'defgeneric', 'defmethod'))),
 
         defun_header: $ =>
-            choice(
+            prec(PREC.SPECIAL, choice(
+                seq(field('keyword', $.defun_keyword),
+                    repeat($._gap),
+                    choice($.unquoting_lit, $.unquote_splicing_lit)
+                ),
                 seq(field('keyword', $.defun_keyword),
                     repeat($._gap),
                     field('function_name', $._form),
@@ -270,12 +289,12 @@ module.exports = grammar(clojure, {
                 seq(field('keyword', alias('lambda', $.defun_keyword)),
                     repeat($._gap),
                     field('lambda_list', choice($.list_lit, $.unquoting_lit)))
-            ),
+            )),
 
         array_dimension: _ => prec(100, /\d+[aA]/),
 
-        char_lit: (_, original) =>
-            seq('#', choice(original, /\\[nN]ewline/, /\\[lL]inefeed/, /\\[Ss]pace/, /\\[nN]ull/, /\\[rR]ull/)),
+        char_lit: _ =>
+            seq('#', /\\([^\f\n\r\t ()]+|[()])/),
 
         vec_lit: $ =>
             prec(PREC.SPECIAL,

src/node-types.json

@@ -601,7 +601,7 @@
       },
       "lambda_list": {
         "multiple": false,
-        "required": true,
+        "required": false,
         "types": [
           {
             "type": "list_lit",
@@ -657,6 +657,14 @@
         {
           "type": "dis_expr",
           "named": true
+        },
+        {
+          "type": "unquote_splicing_lit",
+          "named": true
+        },
+        {
+          "type": "unquoting_lit",
+          "named": true
         }
       ]
     }
@@ -4310,18 +4318,10 @@
             "type": "char_lit",
             "named": true
           },
-          {
-            "type": "comment",
-            "named": true
-          },
           {
             "type": "complex_num_lit",
             "named": true
           },
-          {
-            "type": "dis_expr",
-            "named": true
-          },
           {
             "type": "fancy_literal",
             "named": true
@@ -4840,6 +4840,10 @@
     "type": "until",
     "named": false
   },
+  {
+    "type": "upfrom",
+    "named": false
+  },
   {
     "type": "upto",
     "named": false

test/corpus/basic.txt

@@ -612,9 +612,17 @@ Chars
 ================================================================================
 
 #\?
+#\,
+#\-
+#\:
+#\/
 --------------------------------------------------------------------------------
 
 (source
+  (char_lit)
+  (char_lit)
+  (char_lit)
+  (char_lit)
   (char_lit))
 
 ================================================================================
@@ -767,23 +775,45 @@ Multiple gaps in Loop
                         (sym_lit)))))))))))))
 
 ================================================================================
-Unquoting in defun
+Unquoting in defun 2
 ================================================================================
-
-(defun ,foo ,arguments
-  )
-
+ `((defmacro ,name))
+ `((defmacro ,name))
+ `((defmacro ,name)
+         ,(function-location (macro-function name)))
 --------------------------------------------------------------------------------
 
 (source
-  (list_lit
-    (defun
-      (defun_header
-        (defun_keyword)
-        (unquoting_lit
-          (sym_lit))
-        (unquoting_lit
-          (sym_lit))))))
+  (syn_quoting_lit
+    (list_lit
+      (list_lit
+        (defun
+          (defun_header
+            (defun_keyword)
+            (unquoting_lit
+              (sym_lit)))))))
+  (syn_quoting_lit
+    (list_lit
+      (list_lit
+        (defun
+          (defun_header
+            (defun_keyword)
+            (unquoting_lit
+              (sym_lit)))))))
+  (syn_quoting_lit
+    (list_lit
+      (list_lit
+        (defun
+          (defun_header
+            (defun_keyword)
+            (unquoting_lit
+              (sym_lit)))))
+      (unquoting_lit
+        (list_lit
+          (sym_lit)
+          (list_lit
+            (sym_lit)
+            (sym_lit)))))))
 
 ================================================================================
 Weird quoting
@@ -1125,3 +1155,164 @@ Issue #5 (defpackage)
       (kwd_lit
         (kwd_symbol))
       (package_lit))))
+
+================================================================================
+Issue #6 (#\Replacement_Character)
+================================================================================
+#\Replacement_Character
+--------------------------------------------------------------------------------
+
+(source
+  (char_lit))
+
+================================================================================
+loop with
+================================================================================
+(loop with values)
+(loop with values = a)
+(loop with values
+  finally
+  (return values))
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)))))
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (sym_lit)))))
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)))
+      (loop_clause
+        (termination_clause
+          (list_lit
+            (sym_lit)
+            (sym_lit)))))))
+
+================================================================================
+loop with 2
+================================================================================
+(loop with consecutive-yields fixnum = 0)
+(loop with consecutive-yields fixnum = 0 do
+ (block block))
+
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (sym_lit)
+          (num_lit)))))
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (sym_lit)
+          (num_lit)))
+      (loop_clause
+        (do_clause
+          (list_lit
+            (sym_lit)
+            (sym_lit)))))))
+
+================================================================================
+Loop with keywords (TODO: do should not by type of with!)
+================================================================================
+(loop :with b :do
+   (setq b
+         x)
+   )
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (kwd_lit
+            (kwd_symbol))
+          (list_lit
+            (sym_lit)
+            (sym_lit)
+            (sym_lit)))))))
+
+================================================================================
+Loop with keywords 2
+================================================================================
+
+ (loop
+          for id upfrom 0
+          with frame = (nth-frame-list index))
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (for_clause
+          (sym_lit)
+          (for_clause_word)
+          (num_lit)))
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (list_lit
+            (sym_lit)
+            (sym_lit)))))))
+
+================================================================================
+with type should be symbol or list (TODO!!!)
+================================================================================
+(loop :with b :do
+   (setq b x))
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (loop_macro
+      (loop_clause
+        (with_clause
+          (sym_lit)
+          (kwd_lit
+            (kwd_symbol))
+          (list_lit
+            (sym_lit)
+            (sym_lit)
+            (sym_lit)))))))
+
+================================================================================
+Backslashes in strings
+================================================================================
+  " \( '\), \( \) "
+
+--------------------------------------------------------------------------------
+
+(source
+  (str_lit))
+
+================================================================================
+Chars #\(
+================================================================================
+ (write-char #\( stream)
+
+--------------------------------------------------------------------------------
+
+(source
+  (list_lit
+    (sym_lit)
+    (char_lit)
+    (sym_lit)))

test/sly

@@ -0,0 +1 @@
+Subproject commit 0470c0281498b9de072fcbf3718fc66720eeb3d0