Fix parse error when a comment was inside a string (#42)

* Fix parse error when a comment was inside a string

* Update parser.c

corpus/literals.txt

@@ -60,6 +60,9 @@ b"foo\nbar";
 "foo\
     bar";
 "\"foo\"";
+"/* foo bar */ foo bar";
+"foo\x42\x43bar";
+"foo \x42  \x43 bar";
 
 ---
 
@@ -68,6 +71,9 @@ b"foo\nbar";
   (string_literal)
   (string_literal (escape_sequence))
   (string_literal (escape_sequence))
+  (string_literal (escape_sequence) (escape_sequence))
+  (string_literal)
+  (string_literal (escape_sequence) (escape_sequence))
   (string_literal (escape_sequence) (escape_sequence)))
 
 ============================================

grammar.js

@@ -41,6 +41,7 @@ module.exports = grammar({
   extras: $ => [/\s/, $.line_comment, $.block_comment],
 
   externals: $ => [
+    $._string_content,
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
@@ -1256,7 +1257,7 @@ module.exports = grammar({
       alias(/b?"/, '"'),
       repeat(choice(
         $.escape_sequence,
-        token.immediate(prec(1, /[^\\"]+/))
+        $._string_content
       )),
       token.immediate('"')
     ),

src/grammar.json

@@ -6864,15 +6864,8 @@
                 "name": "escape_sequence"
               },
               {
-                "type": "IMMEDIATE_TOKEN",
-                "content": {
-                  "type": "PREC",
-                  "value": 1,
-                  "content": {
-                    "type": "PATTERN",
-                    "value": "[^\\\\\"]+"
-                  }
-                }
+                "type": "SYMBOL",
+                "name": "_string_content"
               }
             ]
           }
@@ -7237,6 +7230,10 @@
     ]
   ],
   "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "_string_content"
+    },
     {
       "type": "SYMBOL",
       "name": "raw_string_literal"

src/injections.json

@@ -21,6 +21,16 @@
       "id": 1,
       "property_set_id": 0,
       "transitions": [
+        {
+          "type": "macro_invocation",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "macro_rule",
+          "named": true,
+          "state_id": 1
+        },
         {
           "type": "token_tree",
           "named": true,
@@ -32,7 +42,18 @@
     {
       "id": 2,
       "property_set_id": 1,
-      "transitions": [],
+      "transitions": [
+        {
+          "type": "macro_invocation",
+          "named": true,
+          "state_id": 1
+        },
+        {
+          "type": "macro_rule",
+          "named": true,
+          "state_id": 1
+        }
+      ],
       "default_next_state_id": 0
     }
   ],

src/scanner.c

@@ -2,6 +2,7 @@
 #include <wctype.h>
 
 enum TokenType {
+  STRING_CONTENT,
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
@@ -23,6 +24,21 @@ static bool is_num_char(int32_t c) {
 
 bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
                                             const bool *valid_symbols) {
+  if (valid_symbols[STRING_CONTENT]) {
+    bool has_content = false;
+    for (;;) {
+      if (lexer->lookahead == '\"' || lexer->lookahead == '\\') {
+        break;
+      } else if (lexer->lookahead == 0) {
+        return false;
+      }
+      has_content = true;
+      advance(lexer);
+    }
+    lexer->result_symbol = STRING_CONTENT;
+    return has_content;
+  }
+
   while (iswspace(lexer->lookahead)) lexer->advance(lexer, true);
 
   if (