Merge commit '1dabc1c790e07115175057863808085ea60dd08a'

CHANGELOG.md

@@ -6,7 +6,7 @@ Added support for Kotlin and TOML.
 
 Fixed an issue with YAML and `|` block strings.
 
-Updated to the latest upstream C++ and C# parsers.
+Updated to the latest upstream C++, C# and Elixir parsers.
 
 ### Diffing
 

vendor/tree-sitter-elixir/.github/workflows/generate.yml

@@ -22,9 +22,23 @@ jobs:
       - name: Install npm dependencies
         run: npm ci
       - name: Generate parser
-        run: npx tree-sitter generate
+        run: |
+          npx tree-sitter generate
+          npx tree-sitter build-wasm
       - name: Update parser files
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
           commit_message: Generate parser
           file_pattern: src
+      - name: Checkout gh-pages branch to ./gh-pages
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+          path: ./gh-pages
+      - run: mv *.wasm ./gh-pages
+      - name: Update WASM file on gh-pages branch
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Generate WASM
+          file_pattern: "*.wasm"
+          repository: ./gh-pages

vendor/tree-sitter-elixir/.gitignore

@@ -8,3 +8,7 @@
 /build/
 log.html
 tree-sitter-elixir.wasm
+
+# Files generated by Cargo
+/target/
+Cargo.lock

vendor/tree-sitter-elixir/Cargo.toml

@@ -20,7 +20,7 @@ include = [
 path = "bindings/rust/lib.rs"
 
 [dependencies]
-tree-sitter = "0.19"
+tree-sitter = "0.20"
 
 [build-dependencies]
 cc = "1.0"

vendor/tree-sitter-elixir/README.md

@@ -4,6 +4,8 @@
 
 Elixir grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter).
 
+Ready for production. Currently used by GitHub itself for source code highlighting and code navigation.
+
 ## Development
 
 See [the docs](./docs/index.md) for more details.

vendor/tree-sitter-elixir/bindings/rust/build.rs

@@ -22,10 +22,6 @@ fn main() {
     c_config.compile("parser");
     println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
 
-    // If your language uses an external scanner written in C++,
-    // then include this block of code:
-
-    /*
     let mut cpp_config = cc::Build::new();
     cpp_config.cpp(true);
     cpp_config.include(&src_dir);
@@ -36,5 +32,4 @@ fn main() {
     cpp_config.file(&scanner_path);
     cpp_config.compile("scanner");
     println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
-    */
 }

vendor/tree-sitter-elixir/bindings/rust/lib.rs

@@ -35,10 +35,10 @@ pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
 
 // Uncomment these to include any queries that this grammar contains
 
-// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
 // pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
 // pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
-// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
+pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
 
 #[cfg(test)]
 mod tests {

vendor/tree-sitter-elixir/grammar.js

@@ -16,6 +16,7 @@ const PREC = {
   XOR_OP: 140,
   TERNARY_OP: 150,
   CONCAT_OPS: 160,
+  RANGE_OP: 160,
   ADD_OPS: 170,
   MULT_OPS: 180,
   POWER_OP: 190,
@@ -33,13 +34,13 @@ const COMP_OPS = ["==", "!=", "=~", "===", "!=="];
 const REL_OPS = ["<", ">", "<=", ">="];
 const ARROW_OPS = ["|>", "<<<", ">>>", "<<~", "~>>", "<~", "~>", "<~>", "<|>"];
 const IN_OPS = ["in", "not in"];
-const CONCAT_OPS = ["++", "--", "+++", "---", "..", "<>"];
+const CONCAT_OPS = ["++", "--", "+++", "---", "<>"];
 const ADD_OPS = ["+", "-"];
 const MULT_OPS = ["*", "/"];
 const UNARY_OPS = ["+", "-", "!", "^", "~~~", "not"];
 
 const ALL_OPS = [
-  ["->", "when", "::", "|", "=>", "&", "=", "^^^", "//", "**", ".", "@"],
+  ["->", "when", "::", "|", "=>", "&", "=", "^^^", "//", "..", "**", ".", "@"],
   IN_MATCH_OPS,
   OR_OPS,
   AND_OPS,
@@ -213,6 +214,7 @@ module.exports = grammar({
         $.tuple,
         $.bitstring,
         $.map,
+        $._nullary_operator,
         $.unary_operator,
         $.binary_operator,
         $.dot,
@@ -426,6 +428,11 @@ module.exports = grammar({
         )
       ),
 
+    _nullary_operator: ($) =>
+      // Nullary operators don't have any child nodes, so we reuse the
+      // operator_identifier node
+      alias(prec(PREC.RANGE_OP, ".."), $.operator_identifier),
+
     unary_operator: ($) =>
       choice(
         unaryOp($, prec, PREC.CAPTURE_OP, "&", $._capture_expression),
@@ -480,6 +487,7 @@ module.exports = grammar({
         binaryOp($, prec.left, PREC.XOR_OP, "^^^"),
         binaryOp($, prec.right, PREC.TERNARY_OP, "//"),
         binaryOp($, prec.right, PREC.CONCAT_OPS, choice(...CONCAT_OPS)),
+        binaryOp($, prec.right, PREC.RANGE_OP, ".."),
         binaryOp($, prec.left, PREC.ADD_OPS, choice(...ADD_OPS)),
         binaryOp($, prec.left, PREC.MULT_OPS, choice(...MULT_OPS)),
         binaryOp($, prec.left, PREC.POWER_OP, "**"),
@@ -524,6 +532,10 @@ module.exports = grammar({
         alias($._not_in, "not in"),
         "^^",
         ...CONCAT_OPS,
+        // The range operator has both a binary and a nullary version.
+        // The nullary version is already parsed as operator_identifier,
+        // so it covers this case
+        // ".."
         ...MULT_OPS,
         "**",
         "->",
@@ -805,10 +817,13 @@ module.exports = grammar({
       ),
 
     body: ($) =>
-      seq(
-        optional($._terminator),
-        sep1($._expression, $._terminator),
-        optional($._terminator)
+      choice(
+        $._terminator,
+        seq(
+          optional($._terminator),
+          sep1($._expression, $._terminator),
+          optional($._terminator)
+        )
       ),
 
     anonymous_function: ($) =>
@@ -876,11 +891,11 @@ function defineQuoted(start, end, name) {
     [`_quoted_i_${name}`]: ($) =>
       seq(
         field("quoted_start", start),
+        optional(alias($[`_quoted_content_i_${name}`], $.quoted_content)),
         repeat(
-          choice(
-            alias($[`_quoted_content_i_${name}`], $.quoted_content),
-            $.interpolation,
-            $.escape_sequence
+          seq(
+            choice($.interpolation, $.escape_sequence),
+            optional(alias($[`_quoted_content_i_${name}`], $.quoted_content))
           )
         ),
         field("quoted_end", end)
@@ -889,11 +904,12 @@ function defineQuoted(start, end, name) {
     [`_quoted_${name}`]: ($) =>
       seq(
         field("quoted_start", start),
+        optional(alias($[`_quoted_content_${name}`], $.quoted_content)),
         repeat(
-          choice(
-            alias($[`_quoted_content_${name}`], $.quoted_content),
-            // The end delimiter may always be escaped
-            $.escape_sequence
+          seq(
+            // The end delimiter may be escaped in non-interpolating strings too
+            $.escape_sequence,
+            optional(alias($[`_quoted_content_${name}`], $.quoted_content))
           )
         ),
         field("quoted_end", end)

vendor/tree-sitter-elixir/package-lock.json

@@ -5,6 +5,7 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "tree-sitter-elixir",
       "version": "0.19.0",
       "license": "Apache-2.0",
       "dependencies": {
@@ -12,7 +13,7 @@
       },
       "devDependencies": {
         "prettier": "^2.3.2",
-        "tree-sitter-cli": "^0.19.5"
+        "tree-sitter-cli": "^0.20.6"
       }
     },
     "node_modules/nan": {
@@ -33,9 +34,9 @@
       }
     },
     "node_modules/tree-sitter-cli": {
-      "version": "0.19.5",
-      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.19.5.tgz",
-      "integrity": "sha512-kRzKrUAwpDN9AjA3b0tPBwT1hd8N2oQvvvHup2OEsX6mdsSMLmAvR+NSqK9fe05JrRbVvG8mbteNUQsxlMQohQ==",
+      "version": "0.20.6",
+      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.20.6.tgz",
+      "integrity": "sha512-tjbAeuGSMhco/EnsThjWkQbDIYMDmdkWsTPsa/NJAW7bjaki9P7oM9TkLxfdlnm4LXd1wR5wVSM2/RTLtZbm6A==",
       "dev": true,
       "hasInstallScript": true,
       "bin": {
@@ -56,9 +57,9 @@
       "dev": true
     },
     "tree-sitter-cli": {
-      "version": "0.19.5",
-      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.19.5.tgz",
-      "integrity": "sha512-kRzKrUAwpDN9AjA3b0tPBwT1hd8N2oQvvvHup2OEsX6mdsSMLmAvR+NSqK9fe05JrRbVvG8mbteNUQsxlMQohQ==",
+      "version": "0.20.6",
+      "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.20.6.tgz",
+      "integrity": "sha512-tjbAeuGSMhco/EnsThjWkQbDIYMDmdkWsTPsa/NJAW7bjaki9P7oM9TkLxfdlnm4LXd1wR5wVSM2/RTLtZbm6A==",
       "dev": true
     }
   }

vendor/tree-sitter-elixir/package.json

@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "prettier": "^2.3.2",
-    "tree-sitter-cli": "^0.19.5"
+    "tree-sitter-cli": "^0.20.6"
   },
   "tree-sitter": [
     {

vendor/tree-sitter-elixir/queries/tags.scm

@@ -0,0 +1,54 @@
+; Definitions
+
+; * modules and protocols
+(call
+  target: (identifier) @ignore
+  (arguments (alias) @name)
+  (#match? @ignore "^(defmodule|defprotocol)$")) @definition.module
+
+; * functions/macros
+(call
+  target: (identifier) @ignore
+  (arguments
+    [
+      ; zero-arity functions with no parentheses
+      (identifier) @name
+      ; regular function clause
+      (call target: (identifier) @name)
+      ; function clause with a guard clause
+      (binary_operator
+        left: (call target: (identifier) @name)
+        operator: "when")
+    ])
+  (#match? @ignore "^(def|defp|defdelegate|defguard|defguardp|defmacro|defmacrop|defn|defnp)$")) @definition.function
+
+; References
+
+; ignore calls to kernel/special-forms keywords
+(call
+  target: (identifier) @ignore
+  (#match? @ignore "^(def|defp|defdelegate|defguard|defguardp|defmacro|defmacrop|defn|defnp|defmodule|defprotocol|defimpl|defstruct|defexception|defoverridable|alias|case|cond|else|for|if|import|quote|raise|receive|require|reraise|super|throw|try|unless|unquote|unquote_splicing|use|with)$"))
+
+; ignore module attributes
+(unary_operator
+  operator: "@"
+  operand: (call
+    target: (identifier) @ignore))
+
+; * function call
+(call
+  target: [
+   ; local
+   (identifier) @name
+   ; remote
+   (dot
+     right: (identifier) @name)
+  ]) @reference.call
+
+; * pipe into function call
+(binary_operator
+  operator: "|>"
+  right: (identifier) @name) @reference.call
+
+; * modules
+(alias) @name @reference.module

vendor/tree-sitter-elixir/src/node-types.json

@@ -79,6 +79,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true
@@ -177,6 +181,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true
@@ -281,6 +289,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -407,6 +419,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -810,6 +826,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true
@@ -918,6 +938,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1021,6 +1045,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1054,7 +1082,7 @@
     "fields": {},
     "children": {
       "multiple": true,
-      "required": true,
+      "required": false,
       "types": [
         {
           "type": "access_call",
@@ -1128,6 +1156,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1274,6 +1306,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1445,6 +1481,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1556,6 +1596,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true
@@ -1700,6 +1744,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1812,6 +1860,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -1934,6 +1986,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -2060,6 +2116,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -2187,6 +2247,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true
@@ -2395,6 +2459,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -2626,6 +2694,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -2864,6 +2936,10 @@
           "type": "nil",
           "named": true
         },
+        {
+          "type": "operator_identifier",
+          "named": true
+        },
         {
           "type": "quoted_atom",
           "named": true
@@ -2975,6 +3051,10 @@
             "type": "nil",
             "named": true
           },
+          {
+            "type": "operator_identifier",
+            "named": true
+          },
           {
             "type": "quoted_atom",
             "named": true

vendor/tree-sitter-elixir/src/scanner.cc

@@ -174,7 +174,22 @@ int8_t find_quoted_token_info(const bool* valid_symbols) {
 bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
   lexer->result_symbol = info.token_type;
 
+  bool is_heredoc = (info.delimiter_length == 3);
+
   for (bool has_content = false; true; has_content = true) {
+    bool newline = false;
+
+    if (is_newline(lexer->lookahead)) {
+      advance(lexer);
+
+      has_content = true;
+      newline = true;
+
+      while (is_whitespace(lexer->lookahead)) {
+        advance(lexer);
+      }
+    }
+
     lexer->mark_end(lexer);
 
     if (lexer->lookahead == info.end_delimiter) {
@@ -189,7 +204,7 @@ bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
         }
       }
 
-      if (length == info.delimiter_length) {
+      if (length == info.delimiter_length && (!is_heredoc || newline)) {
         return has_content;
       }
     } else {
@@ -199,16 +214,18 @@ bool scan_quoted_content(TSLexer* lexer, const QuotedContentInfo& info) {
           return has_content;
         }
       } else if (lexer->lookahead == '\\') {
-        if (info.supports_interpol) {
+        advance(lexer);
+        if (is_heredoc && lexer->lookahead == '\n') {
+          // We need to know about the newline to correctly recognise
+          // heredoc end delimiter, so we intentionally ignore escaping
+        } else if (info.supports_interpol || lexer->lookahead == info.end_delimiter) {
           return has_content;
-        } else {
-          advance(lexer);
-          if (lexer->lookahead == info.end_delimiter) {
-            return has_content;
-          }
         }
       } else if (lexer->lookahead == '\0') {
-        return false;
+        // If we reached the end of the file, this means there is no
+        // end delimiter, so the syntax is invalid. In that case we
+        // want to treat all the scanned content as quoted content.
+        return has_content;
       } else {
         advance(lexer);
       }

vendor/tree-sitter-elixir/src/tree_sitter/parser.h

@@ -123,6 +123,7 @@ struct TSLanguage {
     unsigned (*serialize)(void *, char *);
     void (*deserialize)(void *, const char *, unsigned);
   } external_scanner;
+  const TSStateId *primary_state_ids;
 };
 
 /*

vendor/tree-sitter-elixir/test/corpus/do_end.txt

@@ -683,6 +683,25 @@ end
         (body
           (integer))))))
 
+=====================================
+stab clause / edge cases / empty right-hand-side
+=====================================
+
+fun do
+  x ->
+end
+
+---
+
+(source
+  (call
+    (identifier)
+    (do_block
+      (stab_clause
+        (arguments
+          (identifier))
+        (body)))))
+
 =====================================
 pattern matching
 =====================================

vendor/tree-sitter-elixir/test/corpus/expression/operator.txt

@@ -656,6 +656,31 @@ x +y +z
             (unary_operator
               (identifier))))))))
 
+=====================================
+nullary range
+=====================================
+
+..
+Enum.to_list(..)
+not ..
+range = ..
+
+---
+
+(source
+  (operator_identifier)
+  (call
+    (dot
+      (alias)
+      (identifier))
+    (arguments
+      (operator_identifier)))
+  (unary_operator
+    (operator_identifier))
+  (binary_operator
+    (identifier)
+    (operator_identifier)))
+
 =====================================
 stepped range
 =====================================

vendor/tree-sitter-elixir/test/corpus/term/string.txt

@@ -1,3 +1,14 @@
+=====================================
+empty
+=====================================
+
+""
+
+---
+
+(source
+  (string))
+
 =====================================
 single line
 =====================================
@@ -171,6 +182,47 @@ this is #{
         (quoted_content)))
     (quoted_content)))
 
+=====================================
+heredoc / delimiter in the middle
+=====================================
+
+"""
+hey """
+"""
+
+---
+
+(source
+  (string
+    (quoted_content)))
+
+=====================================
+heredoc / escaped newline (ignored)
+=====================================
+
+"""
+hey \
+"""
+
+  """
+  hey \
+  """
+
+"""
+hey \
+there
+"""
+
+---
+
+(source
+  (string
+    (quoted_content))
+  (string
+    (quoted_content))
+  (string
+    (quoted_content)))
+
 =====================================
 heredoc / escaped delimiter
 =====================================

vendor/tree-sitter-elixir/test/highlight/anonymous.ex

@@ -41,6 +41,12 @@ fn x, y, z ->
   end
 end
 
+fn ->
+# <- keyword
+#   ^ operator
+end
+# <- keyword
+
 &Set.put(&1, &2)
 # <- operator
 # ^ module

vendor/tree-sitter-elixir/test/highlight/operators.ex

@@ -82,3 +82,8 @@ y = true and false
 #                           ^ module
 #                                 ^ operator
 #                                  ^ function
+
+range = ..
+# <- variable
+#     ^ operator
+#        ^ operator

vendor/tree-sitter-elixir/test/tags/module.ex

@@ -0,0 +1,31 @@
+defmodule Foo.Bar.Baz do
+  #        ^ definition.module
+  #            ^ definition.module
+  #                ^ definition.module
+
+  def init(arg) do
+    #  ^ definition.function
+    state =
+      arg
+      |> map(&(&1 * 2))
+      #   ^ reference.call
+      |> map(&(&1 + 1))
+      #   ^ reference.call
+
+    {:ok, arg}
+  end
+
+  def map(list, fun, acc \\ [])
+  #    ^ definition.function
+
+  def map([head | rest], fun, acc) do
+    #  ^ definition.function
+    map(rest, fun, [fun.(head) | acc])
+    # <- reference.call
+  end
+
+  def map([], _fun, acc), do: Enum.reverse(acc)
+  #    ^ definition.function
+  #                             ^ reference.module
+  #                                    ^ reference.call
+end

vendor/tree-sitter-elixir/test/tags/protocol.ex

@@ -0,0 +1,21 @@
+defprotocol Countable do
+  #          ^ definition.module
+  def count(data)
+  #    ^ definition.function
+end
+
+defimpl Countable, for: Binary do
+  #      ^ reference.module
+  #                      ^ reference.module
+  def count(binary), do: byte_size(binary)
+  #    ^ definition.function
+  #                       ^ reference.call
+end
+
+defimpl Countable, for: List do
+  #      ^ reference.module
+  #                      ^ reference.module
+  def count(list), do: length(list)
+  #    ^ definition.function
+  #                     ^ reference.call
+end