Merge commit 'b338fa9f4807b9e0336cd4dde04948a8c324a4cf'

CHANGELOG.md

@@ -8,7 +8,7 @@ Added support for Ada.
 
 Improved parsing for TOML.
 
-Updated grammar for Clojure.
+Updated grammar for Bash and Clojure.
 
 ### Display
 

vendored_parsers/tree-sitter-bash/.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Build & Test
+name: Build/test
 on:
   push:
     branches:
@@ -7,16 +7,31 @@ on:
     branches:
       - master
 jobs:
-  build-and-test:
+  test:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: true
       matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest]
-      fail-fast: false
+        os: [macos-latest, ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
         with:
-          node-version: 14
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - run: npm install
+      - run: npm test
+  test_windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
       - run: npm install
-      - run: node_modules/.bin/tree-sitter test
+      - run: npm run-script test-windows

vendored_parsers/tree-sitter-bash/.gitignore

@@ -5,3 +5,4 @@ package-lock.json
 test.sh
 Cargo.lock
 target
+.build/

vendored_parsers/tree-sitter-bash/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "examples/bash-it"]
 	path = examples/bash-it
 	url = https://github.com/Bash-it/bash-it.git
+[submodule "examples/bash"]
+	path = examples/bash
+	url = https://git.savannah.gnu.org/git/bash.git

vendored_parsers/tree-sitter-bash/.travis.yml

@@ -1,34 +0,0 @@
-language: node_js
-
-sudo: false
-
-node_js:
-- node
-
-os:
-- linux
-
-matrix:
-  include:
-  - os: osx
-    env: COMPILER=clang++
-    osx_image: xcode9.2
-    compiler: clang
-  - os: linux
-
-branches:
-  only:
-  - master
-  - /^v.*$/
-
-deploy:
-  provider: script
-  script: npm run pre-build && npm run pre-build:upload -u ${PREBUILD_UPLOAD}
-  skip_cleanup: true
-  on:
-    all_branches: true
-    tags: true
-
-env:
-  global:
-    secure: jAdfI37gohXm5VvR5h2AoWVwUMS9hUPxZIP4VR2iOkTSZLaaZczvL04IRNv3bQY6WUYxcVB6xoKbOPMKE1cHvXM8L2W6G1InPxU14P6fKd2MGWxH6PWmCoXHWFQ1AzWkYsMgMokHT2vs4iZ2bLHE9uV3RVuAISO/APsw7AxEpEijljvPkL+Se0nTnCfTbw4ObGxEMRW7TnjKa2W7K8m4QbbGd+r9CzCxBakjdIePQx81IUAAIGwphkY/avNJeUvR1XLnAM38K9Yj0ioSfeV6/QshK/28DLmvilq9sV4LDLVlhgP6h1FB4HW3PEVvYWnWBKa9cUdIAIncya2ibvu2cRHYVbud4Ho6lMCk/QUuiPYLCfnOfa4byVj3DInOY1yCpu+YnqjVOUHV5wQLEvnArLCuJa7dlJSmYquzVr5NkOz9gMmFAyuznL3YdJYY98QH/5GO5FwE8jXiYMoQ+hW52gUB6vZaJqYJu+IwyEhVXj8SVyV3Z77fFmJpdo0FX8R4Mm/3ucVXfWAXifFQbosRM4czVJ3RNTn2Xwf5Vp0ayih4huhwB9reByNPnYJABNUQpiAS3ZRsd+fcmMstkHZOk9EpSq33kubqcN+kub3sGNOJTt9243FOd6BghNNjhC2lqTWZsefyJUuL3Xm5YjJrJWfyif6RPnJZHinn3jfn464=

vendored_parsers/tree-sitter-bash/Cargo.toml

@@ -23,7 +23,7 @@ include = [
 path = "bindings/rust/lib.rs"
 
 [dependencies]
-tree-sitter = "0.19"
+tree-sitter = ">= 0.19, < 0.21"
 
 [build-dependencies]
 cc = "1.0"

vendored_parsers/tree-sitter-bash/Package.swift

@@ -0,0 +1,39 @@
+// swift-tools-version:5.3
+import PackageDescription
+
+let package = Package(
+    name: "TreeSitterBash",
+    products: [
+        .library(name: "TreeSitterBash", targets: ["TreeSitterBash"]),
+    ],
+    dependencies: [],
+    targets: [
+        .target(name: "TreeSitterBash",
+                path: ".",
+                exclude: [
+                    "binding.gyp",
+                    "bindings",
+                    "Cargo.toml",
+                    "corpus",
+                    "grammar.js",
+                    "LICENSE",
+                    "Makefile",
+                    "package.json",
+                    "README.md",
+                    "script",
+                    ".travis.yml",
+                    "appveyor.yaml",
+                    "src/grammar.json",
+                    "src/node-types.json",
+                ],
+                sources: [
+                    "src/parser.c",
+                    "src/scanner.cc",
+                ],
+                resources: [
+                    .copy("queries")
+                ],
+                publicHeadersPath: "bindings/swift",
+                cSettings: [.headerSearchPath("src")])
+    ]
+)

vendored_parsers/tree-sitter-bash/appveyor.yml

@@ -1,7 +1,7 @@
 image: Visual Studio 2015
 
 environment:
-  NODEJS_VERSION: "8"
+  NODEJS_VERSION: "16"
   PREBUILD_UPLOAD:
     secure: oNyyLE7/Oq3TUGZPz6DkLFPUuQzc8FiFS1iuPp7LZ2fyOP/UF4Np4NzJmWcXVyY/
 

vendored_parsers/tree-sitter-bash/bindings/rust/lib.rs

@@ -35,7 +35,7 @@ pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
 
 // Uncomment these to include any queries that this grammar contains
 
-// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+pub const HIGHLIGHT_QUERY: &'static str = include_str!("../../queries/highlights.scm");
 // pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
 // pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
 // pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");

vendored_parsers/tree-sitter-bash/bindings/swift/TreeSitterBash/bash.h

@@ -0,0 +1,16 @@
+#ifndef TREE_SITTER_BASH_H_
+#define TREE_SITTER_BASH_H_
+
+typedef struct TSLanguage TSLanguage;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern TSLanguage *tree_sitter_bash();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_BASH_H_ 

vendored_parsers/tree-sitter-bash/corpus/commands.txt

@@ -265,3 +265,111 @@ usage() {
         (command (command_name (word)))
         (heredoc_redirect (heredoc_start)))
       (heredoc_body (expansion (special_variable_name) (word))))))
+
+==========================================
+Heredocs with empty bodies
+==========================================
+
+node <<JS
+JS
+
+node << 'SJ'
+SJ
+
+usage() {
+	cat <<-EOF
+	EOF
+}
+
+node << 'EOF' > temp
+EOF
+
+---
+
+(program
+ (redirected_statement
+  body: (command
+         name: (command_name
+                (word)))
+  redirect: (heredoc_redirect
+             (heredoc_start)))
+ (heredoc_body)
+ (redirected_statement
+  body: (command
+         name: (command_name
+                (word)))
+  redirect: (heredoc_redirect
+             (heredoc_start)))
+ (heredoc_body)
+ (function_definition
+  name: (word)
+  body: (compound_statement
+         (redirected_statement
+          body: (command
+                 name: (command_name
+                        (word)))
+          redirect: (heredoc_redirect
+                     (heredoc_start)))
+         (heredoc_body)))
+ (redirected_statement
+  body: (command
+         name: (command_name
+                (word)))
+  redirect: (heredoc_redirect
+             (heredoc_start))
+  redirect: (file_redirect
+             destination: (word)))
+ (heredoc_body))
+
+==========================================
+Heredocs with weird characters
+==========================================
+
+node <<_DELIMITER_WITH_UNDERSCORES_
+Hello.
+_DELIMITER_WITH_UNDERSCORES_
+
+node <<'```'
+Hello.
+```
+
+node <<!HEREDOC!
+Hello.
+!HEREDOC!
+
+node <<\'
+Hello.
+'
+
+node <<\\
+Hello.
+\
+
+---
+
+(program
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body)
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body)
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body)
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body)
+  (redirected_statement (command (command_name (word))) (heredoc_redirect (heredoc_start))) (heredoc_body))
+
+==========================================
+Herestrings
+==========================================
+
+node <<< foo
+
+while read -u 3 entry; do
+  echo $entry
+done 3<<<"$ENTRIES"
+
+---
+
+(program
+  (redirected_statement (command (command_name (word))) (herestring_redirect (word)))
+  (redirected_statement
+    (while_statement
+      (command (command_name (word)) (word) (word) (word))
+      (do_group (command (command_name (word)) (simple_expansion (variable_name)))))
+    (herestring_redirect (file_descriptor) (string (simple_expansion (variable_name))))))

vendored_parsers/tree-sitter-bash/corpus/literals.txt

@@ -307,11 +307,11 @@ echo "s$"
 Strings with ANSI-C quoting
 ========================================
 
-echo $'Here\'s johnny!\r\n'
+echo $'Here\'s Johnny!\r\n'
 
 ---
 
-(program (command (command_name (word)) (ansii_c_string)))
+(program (command (command_name (word)) (ansi_c_string)))
 
 =========================================
 Arrays and array expansions

vendored_parsers/tree-sitter-bash/corpus/zsh.txt

@@ -1,35 +0,0 @@
-# See https://zsh.sourceforge.io/Doc/Release/Expansion.html#Parameter-Expansion-Flags
-=============================
-Parameter Expansion Flags
-=============================
-echo ${(v)V}
-echo ${(s.:.)V}
-echo ${(@)V}
-echo ${()V}
-
----
-
-(program
-  (command (command_name (word)) (expansion (expansion_flags) (variable_name)))
-  (command (command_name (word)) (expansion (expansion_flags) (variable_name)))
-  (command (command_name (word)) (expansion (expansion_flags) (variable_name)))
-  (command (command_name (word)) (expansion (expansion_flags) (variable_name))))
-
-=============================
-Parameter Expansion Flags Quotes
-=============================
-echo "${(v_sldkfj_sdklfj)V}"
-
----
-
-(program
-  (command (command_name (word)) (string (expansion (expansion_flags) (variable_name)))))
-=============================
-Parameter Expansion Invalid Flags
-=============================
-echo "${(())V}"
-
----
-
-(program
-(command (command_name (word)) (string (expansion (ERROR) (expansion_flags) (ERROR (word))))))

vendored_parsers/tree-sitter-bash/examples/bash

@@ -0,0 +1 @@
+Subproject commit a99d905216cc0aac5de0c3050f4afc54e21c6bc5

vendored_parsers/tree-sitter-bash/grammar.js

@@ -335,6 +335,7 @@ module.exports = grammar({
     )),
 
     heredoc_redirect: $ => seq(
+      field('descriptor', optional($.file_descriptor)),
       choice('<<', '<<-'),
       $.heredoc_start
     ),
@@ -354,6 +355,7 @@ module.exports = grammar({
     ),
 
     herestring_redirect: $ => seq(
+      field('descriptor', optional($.file_descriptor)),
       '<<<',
       $._literal
     ),
@@ -426,10 +428,10 @@ module.exports = grammar({
       $.word,
       $.string,
       $.raw_string,
-      $.ansii_c_string,
+      $.translated_string,
+      $.ansi_c_string,
       $.expansion,
       $.simple_expansion,
-      $.string_expansion,
       $.command_substitution,
       $.process_substitution
     ),
@@ -468,6 +470,8 @@ module.exports = grammar({
 
     _string_content: $ => token(prec(-1, /([^"`$\\]|\\(.|\r?\n))+/)),
 
+    translated_string: $ => seq('$', $.string),
+
     array: $ => seq(
       '(',
       repeat($._literal),
@@ -476,7 +480,7 @@ module.exports = grammar({
 
     raw_string: $ => /'[^']*'/,
 
-    ansii_c_string: $ => /\$'([^']|\\')*'/,
+    ansi_c_string: $ => /\$'([^']|\\')*'/,
 
     simple_expansion: $ => seq(
       '$',
@@ -488,14 +492,10 @@ module.exports = grammar({
       )
     ),
 
-    string_expansion: $ => seq('$', choice($.string, $.raw_string)),
-
-    // See https://zsh.sourceforge.io/Doc/Release/Expansion.html#Parameter-Expansion-Flags
-    expansion_flags: ($) => seq("(", repeat(/[^()]/), ")"),
+    string_expansion: $ => seq('$', $.string),
 
     expansion: $ => seq(
       '${',
-      optional($.expansion_flags),
       optional(choice('#', '!')),
       optional(choice(
         seq(

vendored_parsers/tree-sitter-bash/package.json

@@ -10,18 +10,15 @@
   "author": "Max Brunsfeld",
   "license": "MIT",
   "dependencies": {
-    "nan": "^2.14.0",
-    "prebuild-install": "^5.3.3"
+    "nan": "^2.17.0"
   },
   "devDependencies": {
-    "prebuild": "^10.0.0",
-    "tree-sitter-cli": "^0.19.1"
+    "tree-sitter-cli": "^0.20.7"
   },
   "scripts": {
-    "install": "prebuild-install || node-gyp rebuild",
-    "pre-build": "prebuild --all --strip --verbose",
-    "pre-build:upload": "prebuild --upload-all",
-    "test": "tree-sitter test && script/parse-examples.sh",
+    "build": "tree-sitter generate && node-gyp build",
+    "parse": "tree-sitter parse",
+    "test": "tree-sitter test && script/parse-examples",
     "test-windows": "tree-sitter test"
   },
   "repository": "https://github.com/tree-sitter/tree-sitter-bash",
@@ -30,8 +27,7 @@
       "scope": "source.bash",
       "file-types": [
         "sh",
-        "bash",
-        "zsh"
+        "bash"
       ]
     }
   ]

vendored_parsers/tree-sitter-bash/queries/highlights.scm

@@ -46,7 +46,6 @@
   ">>"
   "<"
   "|"
-  (expansion_flags)
 ] @operator
 
 (

vendored_parsers/tree-sitter-bash/script/known-failures.txt

@@ -10,6 +10,32 @@ examples/bash-it/plugins/available/git.plugin.bash
 examples/bash-it/plugins/available/go.plugin.bash
 examples/bash-it/themes/colors.theme.bash
 examples/bash-it/themes/doubletime/doubletime.theme.bash
-examples/bash-it/themes/hawaii50/hawaii50.theme.bash
 examples/bash-it/themes/nwinkler_random_colors/nwinkler_random_colors.theme.bash
-examples/bash-it/themes/powerline/powerline.base.bash
+examples/bash/tests/arith-for.tests
+examples/bash/tests/arith.tests
+examples/bash/tests/array.tests
+examples/bash/tests/assoc.tests
+examples/bash/tests/case.tests
+examples/bash/tests/comsub-posix.tests
+examples/bash/tests/comsub.tests
+examples/bash/tests/cond.tests
+examples/bash/tests/dynvar.tests
+examples/bash/tests/errors.tests
+examples/bash/tests/exp.tests
+examples/bash/tests/extglob.tests
+examples/bash/tests/extglob2.tests
+examples/bash/tests/glob.tests
+examples/bash/tests/heredoc.tests
+examples/bash/tests/histexp.tests
+examples/bash/tests/more-exp.tests
+examples/bash/tests/new-exp.tests
+examples/bash/tests/posix2.tests
+examples/bash/tests/posixexp.tests
+examples/bash/tests/posixexp2.tests
+examples/bash/tests/posixpat.tests
+examples/bash/tests/printf.tests
+examples/bash/tests/quote.tests
+examples/bash/tests/quotearray.tests
+examples/bash/tests/redir.tests
+examples/bash/tests/test.tests
+examples/bash/tests/tilde2.tests

vendored_parsers/tree-sitter-bash/script/parse-examples

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+shopt -s globstar
+
+known_failures="$(cat script/known-failures.txt)"
+
+tree-sitter parse -q -t \
+  examples/bash-it/**/*.bash \
+  examples/bash-it/**/*.sh \
+  examples/bash/tests/*.tests \
+  examples/*.sh   \
+  $(for failure in $known_failures; do echo "!${failure}"; done)

vendored_parsers/tree-sitter-bash/script/parse-examples.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-known_failures="$(cat script/known-failures.txt)"
-
-tree-sitter parse -q -t \
-  examples/**/*.bash \
-  examples/**/*.sh   \
-  $(for failure in $known_failures; do echo "!${failure}"; done)

vendored_parsers/tree-sitter-bash/script/update-known-failures.sh

@@ -1,7 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+shopt -s globstar
 
 tree-sitter parse -q -t   \
-  examples/**/*.sh        \
-  examples/**/*.bash      \
+  examples/bash-it/**/*.bash \
+  examples/bash-it/**/*.sh \
+  examples/bash/tests/*.tests \
+  examples/*.sh   \
   | egrep 'ERROR|MISSING' \
   | tee >(cut -d' ' -f1 | sort > script/known-failures.txt)

vendored_parsers/tree-sitter-bash/src/grammar.json

@@ -1449,6 +1449,22 @@
     "heredoc_redirect": {
       "type": "SEQ",
       "members": [
+        {
+          "type": "FIELD",
+          "name": "descriptor",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "file_descriptor"
+              },
+              {
+                "type": "BLANK"
+              }
+            ]
+          }
+        },
         {
           "type": "CHOICE",
           "members": [
@@ -1517,6 +1533,22 @@
     "herestring_redirect": {
       "type": "SEQ",
       "members": [
+        {
+          "type": "FIELD",
+          "name": "descriptor",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "SYMBOL",
+                "name": "file_descriptor"
+              },
+              {
+                "type": "BLANK"
+              }
+            ]
+          }
+        },
         {
           "type": "STRING",
           "value": "<<<"
@@ -1845,19 +1877,19 @@
         },
         {
           "type": "SYMBOL",
-          "name": "ansii_c_string"
+          "name": "translated_string"
         },
         {
           "type": "SYMBOL",
-          "name": "expansion"
+          "name": "ansi_c_string"
         },
         {
           "type": "SYMBOL",
-          "name": "simple_expansion"
+          "name": "expansion"
         },
         {
           "type": "SYMBOL",
-          "name": "string_expansion"
+          "name": "simple_expansion"
         },
         {
           "type": "SYMBOL",
@@ -2063,6 +2095,19 @@
         }
       }
     },
+    "translated_string": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "$"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "string"
+        }
+      ]
+    },
     "array": {
       "type": "SEQ",
       "members": [
@@ -2087,7 +2132,7 @@
       "type": "PATTERN",
       "value": "'[^']*'"
     },
-    "ansii_c_string": {
+    "ansi_c_string": {
       "type": "PATTERN",
       "value": "\\$'([^']|\\\\')*'"
     },
@@ -2139,37 +2184,8 @@
           "value": "$"
         },
         {
-          "type": "CHOICE",
-          "members": [
-            {
-              "type": "SYMBOL",
-              "name": "string"
-            },
-            {
-              "type": "SYMBOL",
-              "name": "raw_string"
-            }
-          ]
-        }
-      ]
-    },
-    "expansion_flags": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "STRING",
-          "value": "("
-        },
-        {
-          "type": "REPEAT",
-          "content": {
-            "type": "PATTERN",
-            "value": "[^()]"
-          }
-        },
-        {
-          "type": "STRING",
-          "value": ")"
+          "type": "SYMBOL",
+          "name": "string"
         }
       ]
     },
@@ -2180,18 +2196,6 @@
           "type": "STRING",
           "value": "${"
         },
-        {
-          "type": "CHOICE",
-          "members": [
-            {
-              "type": "SYMBOL",
-              "name": "expansion_flags"
-            },
-            {
-              "type": "BLANK"
-            }
-          ]
-        },
         {
           "type": "CHOICE",
           "members": [

vendored_parsers/tree-sitter-bash/src/node-types.json

@@ -42,7 +42,7 @@
     "named": true,
     "subtypes": [
       {
-        "type": "ansii_c_string",
+        "type": "ansi_c_string",
         "named": true
       },
       {
@@ -70,7 +70,7 @@
         "named": true
       },
       {
-        "type": "string_expansion",
+        "type": "translated_string",
         "named": true
       },
       {
@@ -638,10 +638,6 @@
           "type": "concatenation",
           "named": true
         },
-        {
-          "type": "expansion_flags",
-          "named": true
-        },
         {
           "type": "regex",
           "named": true
@@ -661,11 +657,6 @@
       ]
     }
   },
-  {
-    "type": "expansion_flags",
-    "named": true,
-    "fields": {}
-  },
   {
     "type": "file_redirect",
     "named": true,
@@ -796,7 +787,18 @@
   {
     "type": "heredoc_redirect",
     "named": true,
-    "fields": {},
+    "fields": {
+      "descriptor": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "file_descriptor",
+            "named": true
+          }
+        ]
+      }
+    },
     "children": {
       "multiple": false,
       "required": true,
@@ -811,7 +813,18 @@
   {
     "type": "herestring_redirect",
     "named": true,
-    "fields": {},
+    "fields": {
+      "descriptor": {
+        "multiple": false,
+        "required": false,
+        "types": [
+          {
+            "type": "file_descriptor",
+            "named": true
+          }
+        ]
+      }
+    },
     "children": {
       "multiple": false,
       "required": true,
@@ -1078,25 +1091,6 @@
       ]
     }
   },
-  {
-    "type": "string_expansion",
-    "named": true,
-    "fields": {},
-    "children": {
-      "multiple": false,
-      "required": true,
-      "types": [
-        {
-          "type": "raw_string",
-          "named": true
-        },
-        {
-          "type": "string",
-          "named": true
-        }
-      ]
-    }
-  },
   {
     "type": "subscript",
     "named": true,
@@ -1197,6 +1191,21 @@
       ]
     }
   },
+  {
+    "type": "translated_string",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "string",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "unary_expression",
     "named": true,
@@ -1537,7 +1546,7 @@
     "named": false
   },
   {
-    "type": "ansii_c_string",
+    "type": "ansi_c_string",
     "named": true
   },
   {

vendored_parsers/tree-sitter-bash/src/scanner.cc

@@ -64,34 +64,21 @@ struct Scanner {
     started_heredoc = false;
     heredoc_delimiter.clear();
 
-    if (lexer->lookahead == '\\') {
-      advance(lexer);
-    }
-
-    int32_t quote = 0;
-    if (heredoc_is_raw || lexer->lookahead == '"') {
-      quote = lexer->lookahead;
-      advance(lexer);
-    }
-
-    while (iswalpha(lexer->lookahead) || (quote != 0 && iswspace(lexer->lookahead))) {
-      heredoc_delimiter += lexer->lookahead;
-      advance(lexer);
-    }
+    string delimiter;
+    bool found_delimiter = advance_word(lexer, delimiter);
+    if (found_delimiter) heredoc_delimiter.assign(delimiter);
 
-    if (lexer->lookahead == quote) {
-      advance(lexer);
-    }
-
-    return !heredoc_delimiter.empty();
+    return found_delimiter;
   }
 
   bool scan_heredoc_end_identifier(TSLexer *lexer) {
     current_leading_word.clear();
     // Scan the first 'n' characters on this line, to see if they match the heredoc delimiter
+    int32_t i = 0;
     while (
       lexer->lookahead != '\0' &&
       lexer->lookahead != '\n' &&
+      ((int32_t)heredoc_delimiter[i++]) == lexer->lookahead &&
       current_leading_word.length() < heredoc_delimiter.length()
     ) {
       current_leading_word += lexer->lookahead;
@@ -159,6 +146,14 @@ struct Scanner {
         }
 
         default: {
+          if (scan_heredoc_end_identifier(lexer)) {
+            heredoc_is_raw = false;
+            started_heredoc = false;
+            heredoc_allows_indent = false;
+            heredoc_delimiter.clear();
+            lexer->result_symbol = end_type;
+            return true;
+          }
           did_advance = true;
           advance(lexer);
           break;
@@ -359,6 +354,40 @@ struct Scanner {
     return false;
   }
 
+  /**
+   * Consume a "word" in POSIX parlance, and returns it unquoted.
+   *
+   * This is an approximate implementation that doesn't deal with any
+   * POSIX-mandated substitution, and assumes the default value for
+   * IFS.
+   */
+  bool advance_word(TSLexer *lexer, string& unquoted_word) {
+    bool empty = true;
+
+    int32_t quote = 0;
+    if (lexer->lookahead == '\'' || lexer->lookahead == '"') {
+      quote = lexer->lookahead;
+      advance(lexer);
+    }
+
+    while (lexer->lookahead && ! (quote ? lexer->lookahead == quote :
+                                  iswspace(lexer->lookahead))) {
+      if (lexer->lookahead == '\\') {
+        advance(lexer);
+        if (! lexer->lookahead) return false;
+      }
+      empty = false;
+      unquoted_word += lexer->lookahead;
+      advance(lexer);
+    }
+
+    if (quote && lexer->lookahead == quote) {
+      advance(lexer);
+    }
+
+    return ! empty;
+  }
+
   string heredoc_delimiter;
   bool heredoc_is_raw;
   bool started_heredoc;

vendored_parsers/tree-sitter-bash/src/tree_sitter/parser.h

@@ -123,6 +123,7 @@ struct TSLanguage {
     unsigned (*serialize)(void *, char *);
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
+  const TSStateId *primary_state_ids;
 };
 
 /*