Use tree-sitter-julia from crates.io

pull/795/head
Wilfred Hughes 2024-12-20 08:25:23 +07:00
parent d2ed2f6123
commit 15b9590db5
36 changed files with 289 additions and 244099 deletions

@ -11,7 +11,7 @@ with YAML.
Improved language detection when one argument is a named pipe.
Updated to the latest tree-sitter parser for C, C++, C#, Haskell,
Java, JavaScript, Objective-C, OCaml, Python, Ruby, Scala and
Java, JavaScript, Julia, Objective-C, OCaml, Python, Ruby, Scala and
TypeScript.
### Syntax Highlighting

11
Cargo.lock generated

@ -255,6 +255,7 @@ dependencies = [
"tree-sitter-haskell",
"tree-sitter-java",
"tree-sitter-javascript",
"tree-sitter-julia",
"tree-sitter-language",
"tree-sitter-objc",
"tree-sitter-ocaml",
@ -1073,6 +1074,16 @@ dependencies = [
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-julia"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4144731a178812ee867619b1e98b3b91e54c1652304b26e5ebe3175b701de323"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-language"
version = "0.1.3"

@ -90,6 +90,7 @@ tree-sitter-c = "0.23.4"
tree-sitter-javascript = "0.23.1"
tree-sitter-typescript = "0.23.2"
tree-sitter-java = "0.23.4"
tree-sitter-julia = "0.23.1"
[dev-dependencies]
# assert_cmd 2.0.10 requires predicates 3.

@ -179,11 +179,6 @@ fn main() {
src_dir: "vendored_parsers/tree-sitter-json-src",
extra_files: vec![],
},
TreeSitterParser {
name: "tree-sitter-julia",
src_dir: "vendored_parsers/tree-sitter-julia-src",
extra_files: vec!["scanner.c"],
},
TreeSitterParser {
name: "tree-sitter-kotlin",
src_dir: "vendored_parsers/tree-sitter-kotlin-src",

@ -131,7 +131,7 @@ sample_files/jsx_1.jsx sample_files/jsx_2.jsx
712ef0cf61b469bbacc84b53b944e5ff -
sample_files/julia_1.jl sample_files/julia_2.jl
8524e48b950e8876bc8f7170e94c7cb5 -
55aa779597d1f6afa681af267706c9b6 -
sample_files/load_1.js sample_files/load_2.js
8defc3cea4d10a8db826973352abe2a1 -

@ -84,7 +84,6 @@ extern "C" {
fn tree_sitter_html() -> ts::Language;
fn tree_sitter_janet_simple() -> ts::Language;
fn tree_sitter_json() -> ts::Language;
fn tree_sitter_julia() -> ts::Language;
fn tree_sitter_kotlin() -> ts::Language;
fn tree_sitter_latex() -> ts::Language;
fn tree_sitter_lua() -> ts::Language;
@ -637,7 +636,9 @@ pub(crate) fn from_language(language: guess::Language) -> TreeSitterConfig {
}
}
Julia => {
let language = unsafe { tree_sitter_julia() };
let language_fn = tree_sitter_julia::LANGUAGE;
let language = tree_sitter::Language::new(language_fn);
TreeSitterConfig {
language: language.clone(),
atom_nodes: vec![

@ -1,199 +1,330 @@
;; Based on the nvim-treesitter highlighting, which is under the Apache license.
;; See https://github.com/nvim-treesitter/nvim-treesitter/blob/cf6d84b5611a3e132d1de5ae1ec2365838143180/queries/julia/highlights.scm
; https://github.com/tree-sitter/tree-sitter-julia/blob/e01c928d11375513138a175a68485c4d53e55ea9/queries/highlights.scm
; MIT license
; Identifiers
(identifier) @variable
(operator) @operator
(range_expression ":" @operator)
(pair_expression "=>" @operator)
;; In case you want type highlighting based on Julia naming conventions (this might collide with mathematical notation)
;((identifier) @type ; exception: mark `A_foo` sort of identifiers as variables
;(match? @type "^[A-Z][^_]"))
;; Disabled for difftastic: predicates are currently ignored.
;; ((identifier) @constant
;; (#match? @constant "^[A-Z][A-Z_]{2}[A-Z_]*$"))
(macro_identifier) @function.macro
(macro_identifier (identifier) @function.macro) ; for any one using the variable highlight
(macro_definition
name: (identifier) @function.macro
["macro" "end" @keyword])
(field_expression
(identifier)
(identifier) @field .)
(identifier) @variable.member .)
(function_definition
name: (identifier) @function)
; Symbols
(quote_expression
":" @string.special.symbol
[
(identifier)
(operator)
] @string.special.symbol)
; Function calls
(call_expression
(identifier) @function)
(identifier) @function.call)
(call_expression
(field_expression (identifier) @method .))
(field_expression
(identifier) @function.call .))
(broadcast_call_expression
(identifier) @function)
(identifier) @function.call)
(broadcast_call_expression
(field_expression (identifier) @method .))
(parameter_list
(identifier) @parameter)
(parameter_list
(optional_parameter .
(identifier) @parameter))
(typed_parameter
(identifier) @parameter
(identifier) @type)
(type_parameter_list
(identifier) @type)
(typed_parameter
(identifier) @parameter
(parameterized_identifier) @type)
(function_expression
. (identifier) @parameter)
(spread_parameter) @parameter
(spread_parameter
(identifier) @parameter)
(named_argument
. (identifier) @parameter)
(argument_list
(typed_expression
(identifier) @parameter
(identifier) @type))
(argument_list
(typed_expression
(identifier) @parameter
(parameterized_identifier) @type))
;; Symbol expressions (:my-wanna-be-lisp-keyword)
(quote_expression
(identifier)) @symbol
(field_expression
(identifier) @function.call .))
;; Parsing error! foo (::Type) gets parsed as two quote expressions
(argument_list
(quote_expression
(quote_expression
(identifier) @type)))
; Macros
(macro_identifier
"@" @function.macro
(_) @function.macro)
(type_argument_list
(identifier) @type)
(parameterized_identifier (_)) @type
(argument_list
(typed_expression . (identifier) @parameter))
(macro_definition
(signature
(call_expression
.
(identifier) @function.macro)))
; Built-in functions
; filter(name -> Base.eval(Core, name) isa Core.Builtin, names(Core))
((identifier) @function.builtin
(#any-of? @function.builtin
"applicable" "fieldtype" "getfield" "getglobal" "invoke" "isa" "isdefined" "modifyfield!"
"modifyglobal!" "nfields" "replacefield!" "replaceglobal!" "setfield!" "setfieldonce!"
"setglobal!" "setglobalonce!" "swapfield!" "swapglobal!" "throw" "tuple" "typeassert" "typeof"))
; Type definitions
(type_head (_) @type.definition)
; Type annotations
(parametrized_type_expression
[
(identifier) @type
(field_expression
(identifier) @type .)
]
(curly_expression
(_) @type))
(typed_expression
(identifier) @type .)
(typed_expression
(parameterized_identifier) @type .)
(abstract_definition
name: (identifier) @type)
(struct_definition
name: (identifier) @type)
(unary_typed_expression
(identifier) @type .)
(subscript_expression
(_)
(range_expression
(identifier) @constant.builtin .)
(#eq? @constant.builtin "end"))
(where_expression
(_) @type .)
(binary_expression
(_) @type
(operator) @operator
(_) @type
(#any-of? @operator "<:" ">:"))
; Built-in types
; filter(name -> typeof(Base.eval(Core, name)) in [DataType, UnionAll], names(Core))
((identifier) @type.builtin
(#any-of? @type.builtin
"AbstractArray" "AbstractChar" "AbstractFloat" "AbstractString" "Any" "ArgumentError" "Array"
"AssertionError" "Bool" "BoundsError" "Char" "ConcurrencyViolationError" "Cvoid" "DataType"
"DenseArray" "DivideError" "DomainError" "ErrorException" "Exception" "Expr" "Float16" "Float32"
"Float64" "Function" "GlobalRef" "IO" "InexactError" "InitError" "Int" "Int128" "Int16" "Int32"
"Int64" "Int8" "Integer" "InterruptException" "LineNumberNode" "LoadError" "Method"
"MethodError" "Module" "NTuple" "NamedTuple" "Nothing" "Number" "OutOfMemoryError"
"OverflowError" "Pair" "Ptr" "QuoteNode" "ReadOnlyMemoryError" "Real" "Ref" "SegmentationFault"
"Signed" "StackOverflowError" "String" "Symbol" "Task" "Tuple" "Type" "TypeError" "TypeVar"
"UInt" "UInt128" "UInt16" "UInt32" "UInt64" "UInt8" "UndefInitializer" "UndefKeywordError"
"UndefRefError" "UndefVarError" "Union" "UnionAll" "Unsigned" "VecElement" "WeakRef"))
; Keywords
[
"const"
"global"
"local"
] @keyword
"end" @keyword
(compound_statement
[
"begin"
"end"
] @keyword)
(quote_statement
[
"quote"
"end"
] @keyword)
(let_statement
[
"let"
"end"
] @keyword)
(if_statement
["if" "end"] @conditional)
[
"if"
"end"
] @keyword.conditional)
(elseif_clause
["elseif"] @conditional)
(else_clause
["else"] @conditional)
(ternary_expression
["?" ":"] @conditional)
"elseif" @keyword.conditional)
(function_definition ["function" "end"] @keyword.function)
(else_clause
"else" @keyword.conditional)
[
"abstract"
"const"
"macro"
"primitive"
"struct"
"type"
] @keyword
(ternary_expression
[
"?"
":"
] @keyword.conditional.ternary)
"return" @keyword.return
(try_statement
[
"try"
"end"
] @keyword.exception)
;; Disabled for difftastic: predicates are currently ignored.
;; ((identifier) @keyword (#any-of? @keyword "global" "local"))
(catch_clause
"catch" @keyword.exception)
(compound_expression
["begin" "end"] @keyword)
(try_statement
["try" "end" ] @exception)
(finally_clause
"finally" @exception)
(catch_clause
"catch" @exception)
(quote_statement
["quote" "end"] @keyword)
(let_statement
["let" "end"] @keyword)
"finally" @keyword.exception)
(for_statement
["for" "end"] @repeat)
(while_statement
["while" "end"] @repeat)
(break_statement) @repeat
(continue_statement) @repeat
[
"for"
"end"
] @keyword.repeat)
(for_binding
"outer" @keyword.repeat)
; comprehensions
(for_clause
"for" @repeat)
"for" @keyword.repeat)
(if_clause
"if" @keyword.conditional)
(while_statement
[
"while"
"end"
] @keyword.repeat)
[
(break_statement)
(continue_statement)
] @keyword.repeat
(function_definition
[
"function"
"end"
] @keyword.function)
(do_clause
["do" "end"] @keyword)
[
"do"
"end"
] @keyword.function)
"in" @keyword.operator
(macro_definition
[
"macro"
"end"
] @keyword)
(return_statement
"return" @keyword.return)
(module_definition
[
"module"
"baremodule"
"end"
] @keyword.import)
(export_statement
["export"] @include)
"export" @keyword.import)
(public_statement
"public" @keyword.import)
(import_statement
["import" "using"] @include)
"import" @keyword.import)
(module_definition
["module" "end"] @include)
(using_statement
"using" @keyword.import)
((identifier) @include (#eq? @include "baremodule"))
(import_alias
"as" @keyword.import)
(selected_import
":" @punctuation.delimiter)
;;; Literals
(struct_definition
[
"mutable"
"struct"
"end"
] @keyword.type)
(integer_literal) @number
(float_literal) @float
(abstract_definition
[
"abstract"
"type"
"end"
] @keyword.type)
(primitive_definition
[
"primitive"
"type"
"end"
] @keyword.type)
; Operators & Punctuation
(operator) @operator
;; Disabled for difftastic: predicates are currently ignored.
;; ((identifier) @float
;; (#any-of? @float "NaN" "NaN16" "NaN32"
;; "Inf" "Inf16" "Inf32"))
(adjoint_expression
"'" @operator)
;; ((identifier) @boolean
;; (#any-of? @boolean "true" "false"))
(range_expression
":" @operator)
;; ((identifier) @constant.builtin
;; (#any-of? @constant.builtin "nothing" "missing"))
(arrow_function_expression
"->" @operator)
[
"."
"..."
"::"
] @punctuation
[
","
";"
] @punctuation.delimiter
[
"("
")"
"["
"]"
"{"
"}"
] @punctuation.bracket
; Keyword operators
((operator) @keyword.operator
(#any-of? @keyword.operator "in" "isa"))
(where_expression
"where" @keyword.operator)
; Built-in constants
((identifier) @constant.builtin
(#any-of? @constant.builtin "nothing" "missing"))
((identifier) @variable.builtin
(#any-of? @variable.builtin "begin" "end")
(#has-ancestor? @variable.builtin index_expression))
; Literals
(boolean_literal) @boolean
(integer_literal) @number
(float_literal) @number.float
((identifier) @number.float
(#any-of? @number.float "NaN" "NaN16" "NaN32" "Inf" "Inf16" "Inf32"))
(character_literal) @character
(escape_sequence) @string.escape
(string_literal) @string
(prefixed_string_literal
prefix: (identifier) @function.macro) @string
(command_literal) @string.special
(prefixed_command_literal
prefix: (identifier) @function.macro) @string.special
((string_literal) @string.documentation
.
[
(abstract_definition)
(assignment)
(const_statement)
(function_definition)
(macro_definition)
(module_definition)
(struct_definition)
])
[
(line_comment)
(block_comment)
] @comment
;;; Punctuation
(quote_expression ":" @symbol)
["::" "." "," "..."] @punctuation.delimiter
["[" "]" "(" ")" "{" "}"] @punctuation.bracket

@ -1 +0,0 @@
tree-sitter-julia/src

@ -1,2 +0,0 @@
/src/** linguist-vendored
/examples/* linguist-vendored

@ -1,28 +0,0 @@
name: Build/test
on:
push:
branches:
- "**"
jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
os: [macos-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16
- run: npm install
- run: npm test
test_windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 16
- run: npm install
- run: npm run-script test-windows

@ -1,8 +0,0 @@
package-lock.json
node_modules
build
*.log
examples/*
!examples/.gitkeep
target
Cargo.lock

@ -1,7 +0,0 @@
corpus
build
script
examples
*.tgz
target
Cargo.lock

@ -1,26 +0,0 @@
[package]
name = "tree-sitter-julia"
description = "julia grammar for the tree-sitter parsing library"
version = "0.19.0"
keywords = ["incremental", "parsing", "julia"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-javascript"
edition = "2018"
license = "MIT"
build = "bindings/rust/build.rs"
include = [
"bindings/rust/*",
"grammar.js",
"queries/*",
"src/*",
]
[lib]
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = "0.19"
[build-dependencies]
cc = "1.0"

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2018 Max Brunsfeld, GitHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

@ -1,10 +0,0 @@
tree-sitter-julia
=================
[![Build/test](https://github.com/tree-sitter/tree-sitter-julia/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/tree-sitter/tree-sitter-julia/actions/workflows/ci.yml)
Julia grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter).
References
* [The Julia Parser](https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm)

@ -1,19 +0,0 @@
{
"targets": [
{
"target_name": "tree_sitter_julia_binding",
"include_dirs": [
"<!(node -e \"require('nan')\")",
"src"
],
"sources": [
"src/parser.c",
"src/scanner.c",
"bindings/node/binding.cc"
],
"cflags_c": [
"-std=c99",
]
}
]
}

@ -1,28 +0,0 @@
#include "tree_sitter/parser.h"
#include <node.h>
#include "nan.h"
using namespace v8;
extern "C" TSLanguage * tree_sitter_julia();
namespace {
NAN_METHOD(New) {}
void Init(Local<Object> exports, Local<Object> module) {
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
tpl->InstanceTemplate()->SetInternalFieldCount(1);
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_julia());
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("julia").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
}
NODE_MODULE(tree_sitter_julia_binding, Init)
} // namespace

@ -1,19 +0,0 @@
try {
module.exports = require("../../build/Release/tree_sitter_julia_binding");
} catch (error1) {
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_julia_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
}
throw error1
}
}
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
} catch (_) {}

@ -1,16 +0,0 @@
fn main() {
let src_dir = std::path::Path::new("src");
let mut c_config = cc::Build::new();
c_config.include(&src_dir);
c_config
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable")
.flag_if_supported("-Wno-trigraphs");
let parser_path = src_dir.join("parser.c");
let scanner_path = src_dir.join("scanner.c");
c_config.file(&parser_path);
c_config.file(&scanner_path);
c_config.compile("parser");
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
}

@ -1,52 +0,0 @@
//! This crate provides julia language support for the [tree-sitter][] parsing library.
//!
//! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code:
//!
//! ```
//! let code = "";
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(tree_sitter_julia::language()).expect("Error loading julia grammar");
//! let tree = parser.parse(code, None).unwrap();
//! ```
//!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
//! [language func]: fn.language.html
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
extern "C" {
fn tree_sitter_julia() -> Language;
}
/// Get the tree-sitter [Language][] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_julia() }
}
/// The content of the [`node-types.json`][] file for this grammar.
///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains
// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
#[cfg(test)]
mod tests {
#[test]
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(super::language())
.expect("Error loading julia language");
}
}

@ -1,902 +0,0 @@
const PREC = [
'assign',
'pair',
'conditional',
'lazy_or',
'lazy_and',
'arrow',
'comparison',
'pipe_left',
'pipe_right',
'colon_quote',
'colon_range',
'plus',
'times',
'rational',
'bitshift',
'power',
'call',
'decl',
'dot',
'postfix',
'prefix',
].reduce((result, name, index) => {
result[name] = index + 10;
return result;
}, {});
const ASSIGN_OPERATORS = `
= += -= *= /= //= \\= ^= ÷= %= <<= >>= >>>= |= &= =
`;
const ARROW_OPERATORS = `
⬿
`;
const COMPARISON_OPERATORS = `
> < >= <= == === != !==
⩿
⪿
`;
const DOTTY_OPERATORS = '… ⁝ ⋮ ⋱ ⋰ ⋯';
const PLUS_OPERATORS = `
+ - | ++ ±
`;
const TIMES_OPERATORS = `
* / ÷ % & × \\
⦿
`;
const BITSHIFT_OPERATORS = '<< >> >>>';
const POWER_OPERATORS = `
^
`;
module.exports =
grammar({
name: 'julia',
word: $ => $.identifier,
inline: $ => [
$._terminator,
$._definition,
$._statement,
],
supertypes: $ => [
$._statement,
$._definition,
$._expression,
$._primary_expression,
],
externals: $ => [
$.block_comment,
$._immediate_paren,
$._string_start,
$._command_start,
$._immediate_string_start,
$._immediate_command_start,
$._string_end,
$._command_end,
$._string_content,
$._string_content_no_interp,
],
conflicts: $ => [
// Arrow functions vs tuples
[$._primary_expression, $.parameter_list],
[$._primary_expression, $.spread_parameter],
[$._primary_expression, $.typed_parameter],
[$._primary_expression, $.named_field],
[$._primary_expression, $.named_field, $.optional_parameter],
[$.named_field, $.optional_parameter],
],
supertypes: $ => [
$._expression,
$._statement,
$._definition,
],
extras: $ => [
/\s/,
$.line_comment,
$.block_comment,
],
rules: {
source_file: $ => optional($._expression_list),
_expression_list: $ => seq(
sep1($._terminator, choice(
$._expression,
$.assignment_expression,
$.bare_tuple_expression
)),
optional($._terminator)
),
// Definitions
_definition: $ => choice(
$.abstract_definition,
$.primitive_definition,
$.struct_definition,
$.module_definition,
$.function_definition,
$.macro_definition
),
function_definition: $ => seq(
'function',
field('name', $.identifier),
field('type_parameters', optional($.type_parameter_list)),
field('parameters', $.parameter_list),
optional($._expression_list),
'end'
),
abstract_definition: $ => seq(
'abstract',
'type',
field('name', $.identifier),
field('type_parameters', optional($.type_parameter_list)),
optional($.subtype_clause),
'end'
),
primitive_definition: $ => seq(
'primitive',
'type',
field('name', $.identifier),
field('type_parameters', optional($.type_parameter_list)),
optional($.subtype_clause),
alias(numeral('0-9'), $.integer_literal),
'end'
),
struct_definition: $ => seq(
optional('mutable'),
'struct',
field('name', $.identifier),
field('type_parameters', optional($.type_parameter_list)),
optional($.subtype_clause),
optional($._expression_list),
'end'
),
module_definition: $ => seq(
'module',
field('name', $.identifier),
optional($._expression_list),
'end'
),
macro_definition: $ => seq(
'macro',
field('name', choice($.identifier, $.operator)),
field('parameters', $.parameter_list),
optional($._expression_list),
'end'
),
parameter_list: $ => seq(
'(',
sep(',', choice(
$.identifier,
$.spread_parameter,
$.optional_parameter,
$.typed_parameter
)),
optional($.keyword_parameters),
')'
),
keyword_parameters: $ => seq(
';',
sep1(',', choice(
$.identifier,
$.spread_parameter,
$.optional_parameter,
$.typed_parameter
))
),
optional_parameter: $ => seq(
choice($.identifier, $.typed_parameter),
'=',
$._expression
),
spread_parameter: $ => seq($.identifier, '...'),
typed_parameter: $ => seq(
$.identifier,
'::',
choice($.identifier, $.parameterized_identifier)
),
type_parameter_list: $ => seq(
'{',
sep1(',', choice($.identifier, $.constrained_parameter)),
'}'
),
constrained_parameter: $ => seq(
field('name', $.identifier),
'<:',
field('value', $._expression)
),
subtype_clause: $ => seq(
'<:',
$._expression
),
// Statements
_statement: $ => choice(
$.if_statement,
$.try_statement,
$.for_statement,
$.while_statement,
$.let_statement,
$.const_statement,
$.quote_statement,
$.break_statement,
$.continue_statement,
$.return_statement,
$.import_statement,
$.export_statement
),
if_statement: $ => seq(
'if',
field('condition', $._expression),
optional($._terminator),
optional($._expression_list),
field('alternative', repeat($.elseif_clause)),
field('alternative', optional($.else_clause)),
'end'
),
elseif_clause: $ => seq(
'elseif',
field('condition', $._expression),
optional($._terminator),
optional($._expression_list)
),
else_clause: $ => seq(
'else',
optional($._expression_list)
),
try_statement: $ => seq(
'try',
optional($._expression_list),
optional($.catch_clause),
optional($.finally_clause),
'end'
),
catch_clause: $ => prec(1, seq(
'catch',
optional($.identifier),
optional($._terminator),
optional($._expression_list),
)),
finally_clause: $ => seq(
'finally',
optional($._terminator),
optional($._expression_list),
),
for_statement: $ => seq(
'for',
sep1(',', $.for_binding),
optional($._terminator),
optional($._expression_list),
'end'
),
while_statement: $ => seq(
'while',
field('condition', $._expression),
optional($._terminator),
optional($._expression_list),
'end'
),
break_statement: $ => 'break',
continue_statement: $ => 'continue',
return_statement: $ => prec.right(-2, seq(
'return',
optional(choice(
$._expression,
$.bare_tuple_expression
))
)),
let_statement: $ => seq(
'let',
sep1(',', $.variable_declaration),
optional($._terminator),
optional($._expression_list),
'end'
),
const_statement: $ => seq(
'const',
prec.right(sep1(',', $.variable_declaration))
),
variable_declaration: $ => prec.right(seq(
$.identifier,
optional(seq('=', $._expression))
)),
quote_statement: $ => seq(
'quote',
optional($._expression_list),
'end'
),
import_statement: $ => prec.right(seq(
choice('using', 'import'),
sep1(',', choice(
$.identifier,
$.scoped_identifier,
$.selected_import
))
)),
selected_import: $ => seq(
choice($.identifier, $.scoped_identifier),
token.immediate(':'),
prec.right(sep1(',', choice(
$.identifier,
$.macro_identifier
)))
),
scoped_identifier: $ => prec(PREC.dot, seq(
optional(choice($.identifier, $.scoped_identifier)),
'.',
$.identifier
)),
export_statement: $ => prec.right(seq(
'export',
sep1(',', $.identifier)
)),
// Expressions
_expression: $ => choice(
$._statement,
$._definition,
$.typed_expression,
$.compound_expression,
$.pair_expression,
alias(':', $.operator),
$.macro_expression,
$.unary_expression,
$.binary_expression,
$.ternary_expression,
$.generator_expression,
$.function_expression,
$.coefficient_expression,
$.spread_expression,
$.range_expression,
$.quote_expression,
$.interpolation_expression,
$._primary_expression,
$._literal,
$.operator,
),
_primary_expression: $ => choice(
$.identifier,
$.array_expression,
$.array_comprehension_expression,
$.matrix_expression,
$.call_expression,
$.field_expression,
$.parenthesized_expression,
$.subscript_expression,
$.parameterized_identifier,
$.tuple_expression,
$.broadcast_call_expression,
),
bare_tuple_expression: $ => prec(-1, seq(
$._expression,
repeat1(prec(-1, seq(',', $._expression)))
)),
operator: $ => choice(
$._comparison_operator,
$._dotty_operator,
$._plus_operator,
$._times_operator,
$._rational_operator,
$._bitshift_operator,
$._power_operator,
$._unary_operator,
),
parenthesized_expression: $ => prec(1, seq(
'(', choice($._expression_list, $.spread_expression), ')'
)),
field_expression: $ => prec(PREC.dot, seq(
$._primary_expression,
'.',
$.identifier
)),
subscript_expression: $ => seq(
choice(
$._primary_expression,
$._literal,
),
token.immediate('['),
sep(',', $._expression),
optional(','),
']'
),
typed_expression: $ => prec(PREC.decl, seq(
$._expression,
choice('::', '<:'),
choice($.identifier, $.parameterized_identifier)
)),
parameterized_identifier: $ => seq(
choice($.identifier, $.field_expression),
$.type_argument_list
),
type_argument_list: $ => seq(
'{',
sep1(',', choice($._expression)),
'}'
),
compound_expression: $ => seq(
'begin',
$._expression_list,
'end'
),
call_expression: $ => prec(PREC.call, seq(
choice($._primary_expression, $.operator),
$._immediate_paren,
choice($.argument_list, $.generator_expression),
optional($.do_clause)
)),
broadcast_call_expression: $ => prec(PREC.call, seq(
$._primary_expression,
'.',
$._immediate_paren,
choice($.argument_list, $.generator_expression),
optional($.do_clause)
)),
macro_expression: $ => prec.right(seq(
$.macro_identifier,
optional(choice(
seq($._immediate_paren, $.argument_list),
$.macro_argument_list
))
)),
macro_argument_list: $ => prec(-1, repeat1(prec(-1, $._expression))),
argument_list: $ => seq(
'(',
sep(',', choice(
$._expression,
alias($.named_field, $.named_argument)
)),
optional(seq(
';',
sep1(',', alias($.named_field, $.named_argument))
)),
optional(','),
')'
),
do_clause: $ => seq(
'do',
$._expression_list,
'end'
),
named_field: $ => seq(
$.identifier,
'=',
$._expression
),
spread_expression: $ => prec(PREC.dot, seq($._expression, '...')),
assignment_expression: $ => prec.right(PREC.assign, seq(
choice(
$._expression,
$.bare_tuple_expression
),
alias(choice($._assign_operator, '='), $.operator),
choice(
$._expression,
$.assignment_expression,
$.bare_tuple_expression
)
)),
unary_expression: $ => choice(
prec(PREC.prefix, seq(
alias($._unary_operator, $.operator),
$._expression,
)),
prec(PREC.postfix, seq($._expression, alias("'", $.operator))),
),
binary_expression: $ => {
const table = [
[prec.left, PREC.power, $._power_operator],
[prec.left, PREC.rational, $._rational_operator],
[prec.left, PREC.bitshift, $._bitshift_operator],
[prec.left, PREC.times, $._times_operator],
[prec.left, PREC.plus, choice('+', $._plus_operator)],
[prec.left, PREC.colon_range, $._dotty_operator],
[prec.right, PREC.arrow, $._arrow_operator],
[prec.right, PREC.pipe_left, '<|'],
[prec.left, PREC.pipe_right, '|>'],
[prec.left, PREC.comparison, choice('in', 'isa', $._comparison_operator)],
[prec.left, PREC.lazy_or, '||'],
[prec.left, PREC.lazy_and, '&&'],
];
return choice(...table.map(([fn, prec, op]) => fn(prec, seq(
$._expression,
alias(op, $.operator),
$._expression,
))));
},
ternary_expression: $ => prec.right(PREC.conditional, seq(
$._expression,
'?',
$._expression,
':',
$._expression
)),
pair_expression: $ => prec.right(PREC.pair, seq(
$._expression,
'=>',
$._expression
)),
tuple_expression: $ => seq(
'(',
choice(
optional(','),
seq(
choice($._expression, $.named_field),
','
),
seq(
choice($._expression, $.named_field),
repeat1(seq(',', choice($._expression, $.named_field))),
optional(',')
)
),
')'
),
array_expression: $ => seq(
'[',
sep(',', $._expression),
optional(','),
']'
),
matrix_expression: $ => prec(-1, seq(
'[',
sep(';', $.matrix_row),
optional(';'),
']'
)),
matrix_row: $ => repeat1(prec(-1, $._expression)),
generator_expression: $ => seq(
'(',
$._expression,
$._comprehension_clause,
')'
),
array_comprehension_expression: $ => seq(
'[',
$._expression,
$._comprehension_clause,
']'
),
_comprehension_clause: $ => seq(
$.for_clause,
repeat(choice(
$.for_clause,
$.if_clause
))
),
if_clause: $ => seq(
'if',
$._expression
),
for_clause: $ => seq(
'for',
sep1(',', $.for_binding)
),
for_binding: $ => seq(
choice($.identifier, $.tuple_expression),
choice('in', '=', '∈'),
$._expression
),
function_expression: $ => prec.right(PREC.arrow,
choice(
seq(
'function',
$.parameter_list,
choice(
$._expression,
$.assignment_expression
),
'end'
),
seq(
choice(
$.identifier,
$.parameter_list,
),
'->',
choice(
$._expression,
$.assignment_expression
)))),
range_expression: $ => prec.left(PREC.colon_range, seq(
$._expression,
':',
$._expression
)),
coefficient_expression: $ => prec(PREC.call, seq(
choice(
alias(numeral('0-9'), $.integer_literal),
$.float_literal,
),
choice(
$.parenthesized_expression,
$.identifier
)
)),
quote_expression: $ => prec.left(PREC.colon_quote, seq(
':',
$._expression
)),
interpolation_expression: $ => prec.left(PREC.colon_quote, seq(
'$',
$._expression
)),
// Tokens
macro_identifier: $ => seq('@', choice(
$.identifier,
$.operator,
alias('.', $.operator)
)),
identifier: $ => {
const operators = [
',',
';',
':',
'(', ')',
'{', '}',
'&',
'$',
ASSIGN_OPERATORS,
ARROW_OPERATORS,
COMPARISON_OPERATORS,
DOTTY_OPERATORS,
PLUS_OPERATORS,
TIMES_OPERATORS,
BITSHIFT_OPERATORS,
POWER_OPERATORS
];
const operatorCharacters = operators
.join(' ')
.trim()
.replace(/\s+/g, '')
.replace(/-/g, '')
.replace(/\\/g, '\\\\')
.replace(/!/g, '');
const start = "[_\\p{L}\\p{Nl}∇]"
const rest = `[^"'\`\\s\\.\\-\\[\\]${operatorCharacters}]*`
return new RegExp(start + rest)
},
// Literals
_literal: $ => choice(
$.integer_literal,
$.float_literal,
$.character_literal,
$.string_literal,
$.command_literal,
$.prefixed_string_literal,
$.prefixed_command_literal,
),
integer_literal: $ => choice(
token(seq('0b', numeral('01'))),
token(seq('0o', numeral('0-7'))),
token(seq('0x', numeral('0-9a-fA-F'))),
numeral('0-9'),
),
float_literal: $ => {
const dec = numeral('0-9');
const hex = numeral('0-9a-fA-F');
const float = seq(
choice(
seq(dec, optional('.'), optional(dec)),
seq('.', dec),
),
optional(/[eEf][+-]?\d+/), // the exponent doesn't allow underscores
)
const hex_float = seq(
choice(
seq('0x', hex, optional('.'), optional(hex)),
seq('0x.', hex),
),
/p[+-]?\d+/, // hex floats must always have an exponent
)
return token(choice(float, hex_float))
},
escape_sequence: $ => token(seq(
'\\',
token.immediate(choice(
/[uU][0-9a-fA-F]{1,6}/, // unicode codepoints
/x[0-9a-fA-F]{2}/,
/["'`$\\abfnrtv]/,
/[0-7]{1,3}/,
)),
)),
character_literal: $ => seq(
"'",
choice(
$.escape_sequence,
/[^'\\]/,
),
"'",
),
string_literal: $ => seq(
$._string_start,
repeat(choice($._string_content, $.string_interpolation, $.escape_sequence)),
$._string_end,
),
command_literal: $ => seq(
$._command_start,
repeat(choice($._string_content, $.string_interpolation, $.escape_sequence)),
$._command_end,
),
prefixed_string_literal: $ => seq(
field('prefix', $.identifier),
$._immediate_string_start,
repeat(choice($._string_content_no_interp, $.escape_sequence)),
$._string_end,
),
prefixed_command_literal: $ => seq(
field('prefix', $.identifier),
$._immediate_command_start,
repeat(choice($._string_content_no_interp, $.escape_sequence)),
$._command_end,
),
string_interpolation: $ => seq(
'$',
choice(
$.identifier,
seq('(', $._expression, ')'),
),
),
_unary_operator: $ => token(addDots('+ - ! ~ ¬ √ ∛ ∜')),
_power_operator: $ => token(addDots(POWER_OPERATORS)),
_bitshift_operator: $ => token(addDots(BITSHIFT_OPERATORS)),
_rational_operator: $ => token(addDots('//')),
_times_operator: $ => token(addDots(TIMES_OPERATORS)),
_plus_operator: $ => token(choice('$', addDots(PLUS_OPERATORS))),
_dotty_operator: $ => token(choice('..', addDots(DOTTY_OPERATORS))),
_comparison_operator: $ => token(choice('<:', '>:', addDots(COMPARISON_OPERATORS))),
_arrow_operator: $ => token(choice('<--', '-->', '<-->', addDots(ARROW_OPERATORS))),
_assign_operator: $ => token(choice(':=', '~', '$=', addDots(ASSIGN_OPERATORS))),
_terminator: $ => choice('\n', ';'),
line_comment: $ => token(seq('#', /.*/))
}
});
function sep(separator, rule) {
return optional(sep1(separator, rule));
}
function sep1(separator, rule) {
return seq(rule, repeat(seq(separator, rule)));
}
function addDots(operatorString) {
const operators = operatorString.trim().split(/\s+/)
return seq(optional('.'), choice(...operators))
}
function numeral(range) {
return RegExp(`[${range}]|([${range}][${range}_]*[${range}])`)
}

@ -1,31 +0,0 @@
{
"name": "tree-sitter-julia",
"version": "0.19.0",
"description": "Julia grammar for tree-sitter",
"main": "bindings/node",
"keywords": [
"parser",
"julia"
],
"author": "Max Brunsfeld",
"license": "MIT",
"dependencies": {
"nan": "^2.14.0"
},
"devDependencies": {
"tree-sitter-cli": "^0.19.1"
},
"scripts": {
"build": "tree-sitter generate && node-gyp build",
"test": "tree-sitter test && script/parse-examples",
"test-windows": "tree-sitter test"
},
"tree-sitter": [
{
"file-types": [
"jl"
],
"scope": "source.julia"
}
]
}

@ -1,190 +0,0 @@
examples/Flux.jl/test/cuda/cuda.jl
examples/Flux.jl/src/losses/Losses.jl
examples/Flux.jl/src/losses/utils.jl
examples/Flux.jl/src/losses/functions.jl
examples/Flux.jl/src/layers/conv.jl
examples/Flux.jl/src/layers/basic.jl
examples/Flux.jl/src/layers/normalise.jl
examples/Flux.jl/src/layers/recurrent.jl
examples/Flux.jl/src/optimise/optimisers.jl
examples/Flux.jl/src/utils.jl
examples/Flux.jl/src/cuda/cudnn.jl
examples/Flux.jl/src/cuda/cuda.jl
examples/Flux.jl/src/cuda/curnn.jl
examples/Flux.jl/src/deprecations.jl
examples/Flux.jl/src/functor.jl
examples/Flux.jl/src/onehot.jl
examples/Flux.jl/src/data/dataloader.jl
examples/Flux.jl/src/data/mnist.jl
examples/Flux.jl/src/data/fashion-mnist.jl
examples/Flux.jl/src/data/Data.jl
examples/Flux.jl/src/data/iris.jl
examples/Flux.jl/src/data/tree.jl
examples/Flux.jl/src/data/sentiment.jl
examples/Flux.jl/src/data/housing.jl
examples/Flux.jl/src/data/cmudict.jl
examples/Flux.jl/src/zeros.jl
examples/Gadfly.jl/test/testscripts/issue1125.jl
examples/Gadfly.jl/test/testscripts/timeseries_month.jl
examples/Gadfly.jl/test/testscripts/contour.jl
examples/Gadfly.jl/test/testscripts/gridstack_empty_args.jl
examples/Gadfly.jl/test/testscripts/unitful_basic.jl
examples/Gadfly.jl/test/testscripts/timeseries_day.jl
examples/Gadfly.jl/test/testscripts/timeseries_year_3.jl
examples/Gadfly.jl/test/testscripts/unitful_geoms.jl
examples/Gadfly.jl/test/testscripts/timeseries_year_1.jl
examples/Gadfly.jl/test/testscripts/percent.jl
examples/Gadfly.jl/test/testscripts/point_shape_coord.jl
examples/Gadfly.jl/test/testscripts/unitful_color.jl
examples/Gadfly.jl/test/runtests.jl
examples/Gadfly.jl/test/regen-precompiles.jl
examples/Gadfly.jl/test/compare_examples.jl
examples/Gadfly.jl/src/guide/keys.jl
examples/Gadfly.jl/src/mapping.jl
examples/Gadfly.jl/src/statistics.jl
examples/Gadfly.jl/src/dataframes.jl
examples/Gadfly.jl/src/scale/scales.jl
examples/Gadfly.jl/src/data.jl
examples/Gadfly.jl/src/Gadfly.jl
examples/Gadfly.jl/src/varset.jl
examples/Gadfly.jl/src/scale.jl
examples/Gadfly.jl/src/bincount.jl
examples/Gadfly.jl/src/geometry.jl
examples/Gadfly.jl/src/coord.jl
examples/Gadfly.jl/src/terminalextensions.jl
examples/Gadfly.jl/src/poetry.jl
examples/Gadfly.jl/src/ticks.jl
examples/Gadfly.jl/src/geom/segment.jl
examples/Gadfly.jl/src/geom/violin.jl
examples/Gadfly.jl/src/geom/blank.jl
examples/Gadfly.jl/src/geom/errorbar.jl
examples/Gadfly.jl/src/geom/hexbin.jl
examples/Gadfly.jl/src/geom/hvabline.jl
examples/Gadfly.jl/src/geom/point.jl
examples/Gadfly.jl/src/geom/bar.jl
examples/Gadfly.jl/src/geom/label.jl
examples/Gadfly.jl/src/geom/ribbon.jl
examples/Gadfly.jl/src/geom/polygon.jl
examples/Gadfly.jl/src/geom/subplot.jl
examples/Gadfly.jl/src/geom/line.jl
examples/Gadfly.jl/src/geom/beeswarm.jl
examples/Gadfly.jl/src/geom/rectbin.jl
examples/Gadfly.jl/src/geom/boxplot.jl
examples/Gadfly.jl/src/aesthetics.jl
examples/Gadfly.jl/src/misc.jl
examples/Gadfly.jl/src/color_misc.jl
examples/Gadfly.jl/src/guide.jl
examples/Gadfly.jl/src/theme.jl
examples/Mocha.jl/tools/image-classifier.jl
examples/Mocha.jl/test/layers/power.jl
examples/Mocha.jl/test/layers/softmax.jl
examples/Mocha.jl/test/layers/inner-product.jl
examples/Mocha.jl/test/layers/memory-output.jl
examples/Mocha.jl/test/layers/tied-inner-product.jl
examples/Mocha.jl/test/layers/crop.jl
examples/Mocha.jl/test/layers/reshape.jl
examples/Mocha.jl/test/layers/channel-pooling.jl
examples/Mocha.jl/test/layers/argmax.jl
examples/Mocha.jl/test/layers/pooling.jl
examples/Mocha.jl/test/layers/random-mask.jl
examples/Mocha.jl/test/layers/element-wise.jl
examples/Mocha.jl/test/layers/gaussian-kl-loss.jl
examples/Mocha.jl/test/layers/index2onehot.jl
examples/Mocha.jl/test/layers/concat.jl
examples/Mocha.jl/test/layers/convolution.jl
examples/Mocha.jl/test/utils/ref-count.jl
examples/Mocha.jl/examples/mnist/mnist-demo.jl
examples/Mocha.jl/examples/mnist/mnist-VAE.jl
examples/Mocha.jl/examples/test-wasserstein.jl
examples/Mocha.jl/examples/cifar10/convert.jl
examples/Mocha.jl/benchmarks/parallel-loop/parallel-pool-module.jl
examples/Mocha.jl/src/parameter.jl
examples/Mocha.jl/src/constraints.jl
examples/Mocha.jl/src/layers/accuracy.jl
examples/Mocha.jl/src/layers/dropout.jl
examples/Mocha.jl/src/layers/power.jl
examples/Mocha.jl/src/layers/softmax.jl
examples/Mocha.jl/src/layers/inner-product.jl
examples/Mocha.jl/src/layers/memory-output.jl
examples/Mocha.jl/src/layers/identity.jl
examples/Mocha.jl/src/layers/split.jl
examples/Mocha.jl/src/layers/tied-inner-product.jl
examples/Mocha.jl/src/layers/square-loss.jl
examples/Mocha.jl/src/layers/wasserstein-loss.jl
examples/Mocha.jl/src/layers/softlabel-softmax-loss.jl
examples/Mocha.jl/src/layers/crop.jl
examples/Mocha.jl/src/layers/hinge-loss.jl
examples/Mocha.jl/src/layers/multinomial-logistic-loss.jl
examples/Mocha.jl/src/layers/binary-cross-entropy-loss.jl
examples/Mocha.jl/src/layers/random-normal.jl
examples/Mocha.jl/src/layers/reshape.jl
examples/Mocha.jl/src/layers/lrn.jl
examples/Mocha.jl/src/layers/binary-accuracy.jl
examples/Mocha.jl/src/layers/channel-pooling.jl
examples/Mocha.jl/src/layers/hdf5-data.jl
examples/Mocha.jl/src/layers/argmax.jl
examples/Mocha.jl/src/layers/async-hdf5-data.jl
examples/Mocha.jl/src/layers/pooling.jl
examples/Mocha.jl/src/layers/random-mask.jl
examples/Mocha.jl/src/layers/memory-data.jl
examples/Mocha.jl/src/layers/element-wise.jl
examples/Mocha.jl/src/layers/gaussian-kl-loss.jl
examples/Mocha.jl/src/layers/hdf5-output.jl
examples/Mocha.jl/src/layers/index2onehot.jl
examples/Mocha.jl/src/layers/concat.jl
examples/Mocha.jl/src/layers/softmax-loss.jl
examples/Mocha.jl/src/layers/convolution.jl
examples/Mocha.jl/src/blob.jl
examples/Mocha.jl/src/pooling-functions.jl
examples/Mocha.jl/src/initializers.jl
examples/Mocha.jl/src/neurons.jl
examples/Mocha.jl/src/cuda/layers/softmax.jl
examples/Mocha.jl/src/cuda/layers/inner-product.jl
examples/Mocha.jl/src/cuda/layers/tied-inner-product.jl
examples/Mocha.jl/src/cuda/layers/pooling.jl
examples/Mocha.jl/src/cuda/layers/convolution.jl
examples/Mocha.jl/src/cuda/cublas.jl
examples/Mocha.jl/src/cuda/blob.jl
examples/Mocha.jl/src/cuda/cudnn.jl
examples/Mocha.jl/src/cuda/cuda.jl
examples/Mocha.jl/src/cuda/neurons.jl
examples/Mocha.jl/src/cuda/utils/math.jl
examples/Mocha.jl/src/cuda/backend.jl
examples/Mocha.jl/src/solvers/adadelta.jl
examples/Mocha.jl/src/solvers/policies.jl
examples/Mocha.jl/src/solvers/adagrad.jl
examples/Mocha.jl/src/solvers/adam.jl
examples/Mocha.jl/src/solvers/sgd.jl
examples/Mocha.jl/src/solvers/nesterov.jl
examples/Mocha.jl/src/utils/ref-count.jl
examples/Mocha.jl/src/utils/blas.jl
examples/Mocha.jl/src/utils/io.jl
examples/Mocha.jl/src/macros.jl
examples/Mocha.jl/src/net.jl
examples/Mocha.jl/src/coffee-break.jl
examples/Mocha.jl/src/regularizers.jl
examples/Mocha.jl/src/coffee/training-summary.jl
examples/Mocha.jl/src/coffee/snapshot.jl
examples/Mocha.jl/src/coffee/validation-performance.jl
examples/Mocha.jl/src/native.jl
examples/Mocha.jl/src/solvers.jl
examples/Mocha.jl/src/exception.jl
examples/Mocha.jl/src/compatibility.jl
examples/Mocha.jl/src/logging.jl
examples/Mocha.jl/src/backend.jl
examples/Mocha.jl/src/data-transformers.jl
examples/IJulia.jl/test/stdio.jl
examples/IJulia.jl/test/inline.jl
examples/IJulia.jl/test/comm.jl
examples/IJulia.jl/deps/kspec.jl
examples/IJulia.jl/deps/build.jl
examples/IJulia.jl/src/hmac.jl
examples/IJulia.jl/src/init.jl
examples/IJulia.jl/src/stdio.jl
examples/IJulia.jl/src/inline.jl
examples/IJulia.jl/src/magics.jl
examples/IJulia.jl/src/comm_manager.jl
examples/IJulia.jl/src/execute_request.jl
examples/IJulia.jl/src/display.jl
examples/IJulia.jl/src/handlers.jl
examples/IJulia.jl/src/IJulia.jl

@ -1,46 +0,0 @@
#!/bin/bash
set -e
function fixture() {
url=$1
sha=$2
name=$(basename $url)
path=examples/$name
echo $name $path
if [ ! -d $path ]; then
git clone --depth=1 $url $path
fi
(
cd $path
git fetch origin $sha
git reset --hard $sha
)
}
fixture https://github.com/FluxML/Flux.jl f3c9ab71bc4ec19bc95f0cd67ac63616254b65a3
fixture https://github.com/pluskid/Mocha.jl 5e15b882d7dd615b0c5159bb6fde2cc040b2d8ee
fixture https://github.com/JuliaLang/IJulia.jl ad459e8d46a908c6b923cafa79572a610e2869bb
fixture https://github.com/GiovineItalia/Gadfly.jl f759d9c76b2a983eff314e307825cbcc7e2e5055
all_examples=$(find "examples" -type f -name '*.jl')
known_failures=$(cat script/known-failures.txt)
examples_to_parse=$(
for example in $all_examples; do
if [[ ! $known_failures == *$example* ]]; then
echo $example
fi
done
)
echo $examples_to_parse | xargs -n 5000 tree-sitter parse -q -t
skipped=$( echo $known_failures | wc -w )
parsed=$( echo $examples_to_parse | wc -w )
total=$(( parsed + skipped ))
percent=$( bc -l <<< "100*$parsed/$total" )
printf "Successfully parsed %d of %d files (%.2f%%)\n" $parsed $total $percent

@ -1,5 +0,0 @@
#!/bin/bash
tree-sitter parse $(find examples -type f -name '*.jl') -q -t \
| egrep 'ERROR|\(' \
| tee >(tr '\t' ' ' | cut -d' ' -f1 > script/known-failures.txt)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,275 +0,0 @@
#include <string.h> // memcpy
#include <tree_sitter/parser.h>
#include <wctype.h>
enum TokenType {
BLOCK_COMMENT,
IMMEDIATE_PAREN,
STRING_START,
COMMAND_START,
IMMEDIATE_STRING_START,
IMMEDIATE_COMMAND_START,
STRING_END,
COMMAND_END,
STRING_CONTENT,
STRING_CONTENT_NO_INTERP,
};
// Block comments and immediate parentheses are easy to parse, but strings
// require extra-attention.
// The main problems that arise when parsing strings are:
// 1. Triple quoted strings allow single quotes inside. e.g. """ "foo" """.
// 2. Strings can have arbitrary interpolations, including other strings.
// e.g. "echo $("foo")"
// 3. Non-standard string literals don't allow interpolations or escape
// sequences, but you can always write \" and \`.
// All of the above also applies to command literals.
// To efficiently store a delimiter, we take advantage of the fact that:
// (int)'"' == 34 && 34 % 2 == 0
// (int)'`' == 96 && 96 % 2 == 0
// i.e. " and ` have an even numeric representation, so we can store a triple
// quoted delimiter as (delimiter + 1).
typedef char Delimiter;
// We use a stack to keep track of the string and command delimiters.
typedef struct {
Delimiter *arr;
unsigned len;
} Stack;
static Stack *new_stack() {
Delimiter *arr = malloc(TREE_SITTER_SERIALIZATION_BUFFER_SIZE);
if (arr == NULL) exit(1);
Stack *stack = malloc(sizeof(Stack));
if (stack == NULL) exit(1);
stack->arr = arr;
stack->len = 0;
return stack;
}
static void free_stack(Stack *stack) {
free(stack->arr);
free(stack);
}
static void push(Stack *stack, char c, bool triple) {
if (stack->len >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) exit(1);
stack->arr[stack->len++] = triple ? (c + 1) : c;
}
static Delimiter pop(Stack *stack) {
if (stack->len == 0) exit(1);
return stack->arr[stack->len--];
}
static unsigned serialize_stack(Stack *stack, char *buffer) {
unsigned len = stack->len;
memcpy(buffer, stack->arr, len);
return len;
}
static void deserialize_stack(Stack *stack, const char *buffer, unsigned len) {
if (len > 0) {
memcpy(stack->arr, buffer, len);
stack->len = len;
} else {
stack->len = 0;
}
}
// Scanner functions
static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
static void mark_end(TSLexer *lexer) { lexer->mark_end(lexer); }
static bool scan_string_start(TSLexer *lexer, Stack *stack, char start_char) {
if (lexer->lookahead != start_char) return false;
advance(lexer);
mark_end(lexer);
for (unsigned count = 1; count < 3; count++) {
if (lexer->lookahead != start_char) {
// It's not a triple quoted delimiter.
push(stack, start_char, false);
return true;
}
advance(lexer);
}
mark_end(lexer);
push(stack, start_char, true);
return true;
}
static bool scan_string_content(TSLexer *lexer, Stack *stack, bool interp) {
if (stack->len == 0) return false; // Stack is empty. We're not in a string.
Delimiter end_char = stack->arr[stack->len - 1]; // peek
bool is_triple = false;
bool has_content = false;
if (end_char % 2 != 0) {
is_triple = true;
end_char--;
}
TSSymbol end_symbol = (end_char == '"') ? STRING_END : COMMAND_END;
TSSymbol end_content = interp ? STRING_CONTENT : STRING_CONTENT_NO_INTERP;
while (lexer->lookahead) {
if (interp && (lexer->lookahead == '$' || lexer->lookahead == '\\')) {
mark_end(lexer);
lexer->result_symbol = end_content;
return has_content;
} else if (lexer->lookahead == '\\') {
mark_end(lexer);
advance(lexer);
// Only escape if there's a delimiter character
if (lexer->lookahead == end_char) {
lexer->result_symbol = end_content;
return has_content;
} else {
mark_end(lexer);
}
} else if (lexer->lookahead == end_char) {
if (is_triple) {
mark_end(lexer);
for (unsigned count = 1; count < 3; count++) {
advance(lexer);
if (lexer->lookahead != end_char) {
mark_end(lexer);
lexer->result_symbol = end_content;
return true;
}
}
}
if (has_content) {
lexer->result_symbol = end_content;
} else {
pop(stack);
advance(lexer);
mark_end(lexer);
lexer->result_symbol = end_symbol;
}
return true;
}
advance(lexer);
has_content = true;
}
return false;
}
static bool scan_block_comment(TSLexer *lexer) {
if (lexer->lookahead != '#') return false;
advance(lexer);
if (lexer->lookahead != '=') return false;
advance(lexer);
bool after_eq = false;
unsigned nesting_depth = 1;
for (;;) {
switch (lexer->lookahead) {
case '=':
advance(lexer);
after_eq = true;
break;
case '#':
advance(lexer);
if (after_eq) {
after_eq = false;
nesting_depth--;
if (nesting_depth == 0) {
lexer->result_symbol = BLOCK_COMMENT;
return true;
}
} else {
after_eq = false;
if (lexer->lookahead == '=') {
nesting_depth++;
advance(lexer);
}
}
break;
case '\0':
return false;
default:
advance(lexer);
after_eq = false;
break;
}
}
}
bool tree_sitter_julia_external_scanner_scan(
void *payload,
TSLexer *lexer,
const bool *valid_symbols
) {
if (lexer->lookahead == '(' && valid_symbols[IMMEDIATE_PAREN]) {
lexer->result_symbol = IMMEDIATE_PAREN;
return true;
}
if (valid_symbols[IMMEDIATE_STRING_START] &&
scan_string_start(lexer, payload, '"')) {
lexer->result_symbol = IMMEDIATE_STRING_START;
return true;
}
if (valid_symbols[IMMEDIATE_COMMAND_START] &&
scan_string_start(lexer, payload, '`')) {
lexer->result_symbol = IMMEDIATE_COMMAND_START;
return true;
}
// content or end
if (valid_symbols[STRING_CONTENT] &&
scan_string_content(lexer, payload, true)) {
return true;
}
if (valid_symbols[STRING_CONTENT_NO_INTERP] &&
scan_string_content(lexer, payload, false)) {
return true;
}
// Ignore whitespace
while (iswspace(lexer->lookahead)) {
lexer->advance(lexer, true);
}
if (valid_symbols[STRING_START] && scan_string_start(lexer, payload, '"')) {
lexer->result_symbol = STRING_START;
return true;
}
if (valid_symbols[COMMAND_START] && scan_string_start(lexer, payload, '`')) {
lexer->result_symbol = COMMAND_START;
return true;
}
if (valid_symbols[BLOCK_COMMENT] && scan_block_comment(lexer)) {
return true;
}
return false;
}
void *tree_sitter_julia_external_scanner_create() { return new_stack(); }
void tree_sitter_julia_external_scanner_destroy(void *payload) {
free_stack(payload);
}
unsigned tree_sitter_julia_external_scanner_serialize(
void *payload,
char *buffer
) {
return serialize_stack(payload, buffer);
}
void tree_sitter_julia_external_scanner_deserialize(
void *payload,
const char *buffer,
unsigned length
) {
deserialize_stack(payload, buffer, length);
}

@ -1,224 +0,0 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
/*
* Lexer Macros
*/
#define START_LEXER() \
bool result = false; \
bool skip = false; \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_

@ -1,198 +0,0 @@
=================
Modules
=================
module A
module B
end
module C
end
end
---
(source_file
(module_definition (identifier)
(module_definition (identifier))
(module_definition (identifier))))
=================
Structs
=================
# primitive
primitive type T 8 end
primitive type T <: S 16 end
primitive type Ptr{T} 32 end
# abstract
abstract type T end
abstract type T <: S end
abstract type T{S} <: U end
# empty
struct NoFields
end
# mutable
mutable struct Bar
baz
qux::Float64
end
# parametric
struct Point{T}
x::T
y::T
end
# parametric with constraints, parent type
struct Rational{T<:Integer} <: Real
num::T
den::T
end
# parametric fields
mutable struct Foo<:Bar
asd::Vector{Bar}
end
---
(source_file
(line_comment)
(primitive_definition
(identifier)
(integer_literal))
(primitive_definition
(identifier)
(subtype_clause (identifier))
(integer_literal))
(primitive_definition
(identifier)
(type_parameter_list
(identifier))
(integer_literal))
(line_comment)
(abstract_definition
(identifier))
(abstract_definition
(identifier)
(subtype_clause (identifier)))
(abstract_definition
(identifier)
(type_parameter_list
(identifier))
(subtype_clause (identifier)))
(line_comment)
(struct_definition
(identifier))
(line_comment)
(struct_definition
(identifier)
(identifier)
(typed_expression (identifier) (identifier)))
(line_comment)
(struct_definition
(identifier)
(type_parameter_list (identifier))
(typed_expression (identifier) (identifier))
(typed_expression (identifier) (identifier)))
(line_comment)
(struct_definition
(identifier)
(type_parameter_list (constrained_parameter (identifier) (identifier)))
(subtype_clause (identifier))
(typed_expression (identifier) (identifier))
(typed_expression (identifier) (identifier)))
(line_comment)
(struct_definition
(identifier)
(subtype_clause (identifier))
(typed_expression
(identifier)
(parameterized_identifier (identifier) (type_argument_list (identifier))))))
===============================
Functions with typed parameters
===============================
function a(b::C)
end
function d{E}(f::E)
end
---
(source_file
(function_definition
(identifier)
(parameter_list (typed_parameter (identifier) (identifier))))
(function_definition
(identifier)
(type_parameter_list (identifier))
(parameter_list (typed_parameter (identifier) (identifier)))))
===============================================
Functions with optional and keyword parameters
===============================================
function a(b::C = 1; d, e::F = 2)
end
---
(source_file
(function_definition
(identifier)
(parameter_list
(optional_parameter (typed_parameter (identifier) (identifier)) (integer_literal))
(keyword_parameters
(identifier)
(optional_parameter
(typed_parameter (identifier) (identifier))
(integer_literal))))))
============================
Macro definitions
======================
macro treelike(T, fs = nothing)
fs = [:($(map(QuoteNode, fs.args)...),)]
:(treelike(@__MODULE__, $(esc(T)), $(fs...)))
end
---
(source_file
(macro_definition
(identifier)
(parameter_list (identifier) (optional_parameter (identifier) (identifier)))
(assignment_expression
(identifier)
(operator)
(array_expression
(quote_expression
(tuple_expression
(interpolation_expression
(parenthesized_expression (spread_expression
(call_expression
(identifier)
(argument_list
(identifier)
(field_expression (identifier) (identifier)))))))))))
(quote_expression
(parenthesized_expression
(call_expression
(identifier)
(argument_list
(macro_expression (macro_identifier (identifier)))
(interpolation_expression (parenthesized_expression
(call_expression (identifier) (argument_list (identifier)))))
(interpolation_expression (parenthesized_expression
(spread_expression (identifier))))))))))

@ -1,316 +0,0 @@
==============================
identifiers
==============================
abc_123_ABC
_fn!
ρ; φ; z
x
θ̄
logŷ
ϵ
ŷ
---
(source_file
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier)
(identifier))
=================
Functions
=================
function test(x, y)
end
---
(source_file
(function_definition
(identifier)
(parameter_list (identifier) (identifier))))
=================
Function calls
=================
a()
b("hi", 2)
c(d...)
d(e; f = g)
new{typeof(xs)}(xs)
---
(source_file
(call_expression (identifier) (argument_list))
(call_expression (identifier) (argument_list (string_literal) (integer_literal)))
(call_expression (identifier) (argument_list (spread_expression (identifier))))
(call_expression (identifier) (argument_list (identifier) (named_argument (identifier) (identifier))))
(call_expression
(parameterized_identifier
(identifier)
(type_argument_list
(call_expression (identifier) (argument_list (identifier)))))
(argument_list (identifier))))
=============================
Function calls with do blocks
=============================
a(b) do
print(c)
end
---
(source_file
(call_expression
(identifier)
(argument_list (identifier))
(do_clause
(call_expression (identifier) (argument_list (identifier))))))
=====================
Fields and subscripts
=====================
(a[1].b().c).d
---
(source_file
(field_expression
(parenthesized_expression (field_expression
(call_expression
(field_expression
(subscript_expression (identifier) (integer_literal))
(identifier))
(argument_list))
(identifier)))
(identifier)))
=================
Macro calls
=================
@assert x == y
@assert x == y "a message"
@. a
@testset "a" begin
b = c
end
---
(source_file
(macro_expression
(macro_identifier (identifier))
(macro_argument_list
(binary_expression (identifier) (operator) (identifier))))
(macro_expression
(macro_identifier (identifier))
(macro_argument_list
(binary_expression (identifier) (operator) (identifier))
(string_literal)))
(macro_expression (macro_identifier (operator)) (macro_argument_list (identifier)))
(macro_expression
(macro_identifier (identifier))
(macro_argument_list
(string_literal)
(compound_expression (assignment_expression (identifier) (operator) (identifier))))))
===========================
Coefficient expressions
===========================
2x^2 - 3x + 1
1.5x^2 - .5x + 1
2^2x
2(x-1)^2 - 3(x-1) + 1
---
(source_file
(binary_expression
(binary_expression
(binary_expression
(coefficient_expression (integer_literal) (identifier))
(operator)
(integer_literal))
(operator)
(coefficient_expression (integer_literal) (identifier)))
(operator)
(integer_literal))
(binary_expression
(binary_expression
(binary_expression
(coefficient_expression (float_literal) (identifier))
(operator)
(integer_literal))
(operator)
(coefficient_expression (float_literal) (identifier)))
(operator)
(integer_literal))
(binary_expression
(integer_literal)
(operator)
(coefficient_expression (integer_literal) (identifier)))
(binary_expression
(binary_expression
(binary_expression
(coefficient_expression
(integer_literal)
(parenthesized_expression (binary_expression (identifier) (operator) (integer_literal))))
(operator)
(integer_literal))
(operator)
(coefficient_expression
(integer_literal)
(parenthesized_expression (binary_expression (identifier) (operator) (integer_literal)))))
(operator)
(integer_literal)))
=================
Tuples
=================
()
(1)
(1,)
(2,3,4,)
---
(source_file
(tuple_expression)
(parenthesized_expression (integer_literal))
(tuple_expression (integer_literal))
(tuple_expression (integer_literal) (integer_literal) (integer_literal)))
=================
Named tuples
=================
(a = 1)
(a = 1,)
(a = 1, b = 2)
---
(source_file
(parenthesized_expression
(assignment_expression (identifier) (operator) (integer_literal)))
(tuple_expression
(named_field (identifier) (integer_literal)))
(tuple_expression
(named_field (identifier) (integer_literal))
(named_field (identifier) (integer_literal))))
=================
Arrays
=================
[]
[1, 2,]
---
(source_file
(array_expression)
(array_expression (integer_literal) (integer_literal)))
=================
Matrices
=================
[1 2]
[1 2; 3 4]
[1 2; 3 4;]
---
(source_file
(matrix_expression
(matrix_row (integer_literal) (integer_literal)))
(matrix_expression
(matrix_row (integer_literal) (integer_literal))
(matrix_row (integer_literal) (integer_literal)))
(matrix_expression
(matrix_row (integer_literal) (integer_literal))
(matrix_row (integer_literal) (integer_literal))))
====================
Function expressions
====================
x -> x^2
(x,y,z)->2x+y-z
()->3
function () 3 end
() -> (sleep(0.1); i += 1; l)
---
(source_file
(function_expression
(identifier)
(binary_expression (identifier) (operator) (integer_literal)))
(function_expression
(parameter_list (identifier) (identifier) (identifier))
(binary_expression
(binary_expression
(coefficient_expression (integer_literal) (identifier))
(operator)
(identifier))
(operator)
(identifier)))
(function_expression
(parameter_list)
(integer_literal))
(function_expression
(parameter_list)
(integer_literal))
(function_expression
(parameter_list)
(parenthesized_expression
(call_expression (identifier) (argument_list (float_literal)))
(assignment_expression (identifier) (operator) (integer_literal))
(identifier))))
============================
Generator expressions
============================
(b(c, e) for c in d, e = 5 if e)
[b(c, e) for c in d for e in f]
---
(source_file
(generator_expression
(call_expression (identifier) (argument_list (identifier) (identifier)))
(for_clause
(for_binding (identifier) (identifier))
(for_binding (identifier) (integer_literal)))
(if_clause (identifier)))
(array_comprehension_expression
(call_expression (identifier) (argument_list (identifier) (identifier)))
(for_clause (for_binding (identifier) (identifier)))
(for_clause (for_binding (identifier) (identifier)))))

@ -1,181 +0,0 @@
==============================
integer number literals
==============================
0b01
0o01234567
0123456789
123_456_789
0x0123456789_abcdef_ABCDEF
---
(source_file
(integer_literal)
(integer_literal)
(integer_literal)
(integer_literal)
(integer_literal))
==============================
float number literals
==============================
0123456789.
.0123456789
0123456789.0123456789
9e10
9E-1
9f10
9f-1
0x0123456789_abcdef.ABCDEFp0
0x0123456789_abcdef_ABCDEF.p-1
0x.0123456789_abcdef_ABCDEFp1
----
(source_file
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal)
(float_literal))
==============================
character literals
==============================
' '
'o'
'\t'
'\uffff'
'\U10ffff'
---
(source_file
(character_literal)
(character_literal)
(character_literal (escape_sequence))
(character_literal (escape_sequence))
(character_literal (escape_sequence)))
==============================
string literals
==============================
""
"\""
"foo
bar"
"this is a \"string\"."
"""this is also a "string"."""
band = "Interpol"
"$band is a cool band"
"$(2π) is a cool number"
"cells interlinked within $("cells interlinked whithin $("cells interlinked whithin one stem")")"
---
(source_file
(string_literal)
(string_literal (escape_sequence))
(string_literal)
(string_literal (escape_sequence) (escape_sequence))
(string_literal)
(assignment_expression
(identifier)
(operator)
(string_literal))
(string_literal
(string_interpolation (identifier)))
(string_literal
(string_interpolation (coefficient_expression (integer_literal) (identifier))))
(string_literal
(string_interpolation
(string_literal
(string_interpolation
(string_literal))))))
==============================
command string literals
==============================
`pwd`
m`pwd`
`cd $dir`
`echo \`cmd\``
```
echo "\033[31mred\033[m"
```
---
(source_file
(command_literal)
(prefixed_command_literal
prefix: (identifier))
(command_literal (string_interpolation (identifier)))
(command_literal (escape_sequence) (escape_sequence))
(command_literal
(escape_sequence)
(escape_sequence)))
==============================
non-standard string literals
==============================
trailing_ws = r"\s+$"
version = v"1.0"
---
(source_file
(assignment_expression
(identifier)
(operator)
(prefixed_string_literal
prefix: (identifier)))
(assignment_expression
(identifier)
(operator)
(prefixed_string_literal
prefix: (identifier))))
==============================
comments
==============================
# comment
#= comment =#
#=
comment
=#
x = #= comment =# 1
#=
nested #= comments =# =#
#==#
---
(source_file
(line_comment)
(block_comment)
(block_comment)
(assignment_expression (identifier) (operator) (block_comment) (integer_literal))
(block_comment)
(block_comment))

@ -1,255 +0,0 @@
==============================
assignment operators
==============================
a = b
a .. b = a * b
c &= d ÷= e
tup = 1, 2, 3
car, cdr... = list
---
(source_file
(assignment_expression
(identifier)
(operator)
(identifier))
(assignment_expression
(binary_expression (identifier) (operator) (identifier))
(operator)
(binary_expression (identifier) (operator) (identifier)))
(assignment_expression
(identifier)
(operator)
(assignment_expression
(identifier)
(operator)
(identifier)))
(assignment_expression
(identifier)
(operator)
(bare_tuple_expression
(integer_literal)
(integer_literal)
(integer_literal)))
(assignment_expression
(bare_tuple_expression
(identifier)
(spread_expression (identifier)))
(operator)
(identifier)))
==============================
binary operators
==============================
a + b
a ++ 1 × b ⥌ 2 → c
a:(a // b)
x = A \ (v × w)
a & b | c
(x >>> 16, x >>> 8, x) .& 0xff
---
(source_file
; Sanity check
(binary_expression (identifier) (operator) (identifier))
; plus/times/power/arrow
; (→ (++ a (× 1 (⥌ b 2))) c)
(binary_expression
(binary_expression
(identifier)
(operator)
(binary_expression
(integer_literal)
(operator)
(binary_expression
(identifier)
(operator)
(integer_literal))))
(operator)
(identifier))
; range/rational
(range_expression
(identifier)
(parenthesized_expression
(binary_expression
(identifier)
(operator)
(identifier))))
; LA
(assignment_expression
(identifier)
(operator)
(binary_expression
(identifier)
(operator)
(parenthesized_expression
(binary_expression
(identifier)
(operator)
(identifier)))))
; bitwise
(binary_expression
(binary_expression (identifier) (operator) (identifier))
(operator)
(identifier))
(binary_expression
(tuple_expression
(binary_expression (identifier) (operator) (integer_literal))
(binary_expression (identifier) (operator) (integer_literal))
(identifier))
(operator)
(integer_literal)))
==============================
binary comparison operators
==============================
a === 1
a! != 0
A ⊆ B ⊆ C
x ≥ 0 ≥ z
---
(source_file
(binary_expression (identifier) (operator) (integer_literal))
(binary_expression (identifier) (operator) (integer_literal))
; Chained comparisons are parsed as a single expression in Julia.
; So this isn't 100% correct.
(binary_expression
(binary_expression
(identifier)
(operator)
(identifier))
(operator)
(identifier))
(binary_expression
(binary_expression
(identifier)
(operator)
(integer_literal))
(operator)
(identifier)))
==============================
pair operator
==============================
Dict(b => c, d => e)
---
(source_file
(call_expression
(identifier)
(argument_list
(pair_expression (identifier) (identifier))
(pair_expression (identifier) (identifier)))))
==============================
unary operators
==============================
+a
-b
√9
[a, b]'
!p === !(p)
1 ++ +2
---
(source_file
(unary_expression (operator) (identifier))
(unary_expression (operator) (identifier))
(unary_expression (operator) (integer_literal))
(unary_expression (array_expression (identifier) (identifier)) (operator))
(binary_expression
(unary_expression (operator) (identifier))
(operator)
(call_expression (operator) (argument_list (identifier))))
(binary_expression
(integer_literal)
(operator)
(unary_expression (operator) (integer_literal))))
=============================
operator broadcasting
=============================
a .* b .+ c
.~[x]
---
(source_file
(binary_expression
(binary_expression (identifier) (operator) (identifier))
(operator)
(identifier))
(unary_expression
(operator)
(array_expression (identifier))))
==============================
ternary operator
==============================
x = batch_size == 1 ?
rand(10) :
rand(10, batch_size)
---
(source_file
(assignment_expression
(identifier)
(operator)
(ternary_expression
(binary_expression (identifier) (operator) (integer_literal))
(call_expression (identifier) (argument_list (integer_literal)))
(call_expression (identifier) (argument_list (integer_literal) (identifier))))))
==============================
operators as values
==============================
x = +
⪯ = .≤
print(:)
foo(^, ÷, -)
---
(source_file
(assignment_expression
(identifier)
(operator)
(operator))
(assignment_expression
(operator)
(operator)
(operator))
(call_expression
(identifier)
(argument_list (operator)))
(call_expression
(identifier)
(argument_list (operator) (operator) (operator))))

@ -1,216 +0,0 @@
=================
If statements
=================
if a
b()
elseif c
d()
d()
else
e()
end
# empty bodies
if a
elseif b
else
end
---
(source_file
(if_statement
(identifier)
(call_expression (identifier) (argument_list))
(elseif_clause
(identifier)
(call_expression (identifier) (argument_list))
(call_expression (identifier) (argument_list)))
(else_clause
(call_expression (identifier) (argument_list))))
(line_comment)
(if_statement
(identifier)
(elseif_clause
(identifier))
(else_clause)))
===============================
For statements
===============================
for i in [1, 2, 3]
print(i)
end
for (a, b) in c
print(a, b)
end
---
(source_file
(for_statement
(for_binding
(identifier)
(array_expression (integer_literal) (integer_literal) (integer_literal)))
(call_expression (identifier) (argument_list (identifier))))
(for_statement
(for_binding
(tuple_expression (identifier) (identifier))
(identifier))
(call_expression (identifier) (argument_list (identifier) (identifier)))))
===============================
While statements
===============================
while i < 5
print(i)
continue
break
end
while a(); b(); end
---
(source_file
(while_statement
(binary_expression (identifier) (operator) (integer_literal))
(call_expression (identifier) (argument_list (identifier)))
(continue_statement)
(break_statement))
(while_statement
(call_expression (identifier) (argument_list))
(call_expression (identifier) (argument_list))))
==============================
Return statements
==============================
return
return a
return a || b
return a, b, c
---
(source_file
(return_statement)
(return_statement (identifier))
(return_statement (binary_expression (identifier) (operator) (identifier)))
(return_statement (bare_tuple_expression (identifier) (identifier) (identifier))))
===============================
Let statements
===============================
let var1 = value1, var2, var3 = value3
code
end
---
(source_file
(let_statement
(variable_declaration (identifier) (identifier))
(variable_declaration (identifier))
(variable_declaration (identifier) (identifier))
(identifier)))
===============================
Const statements
===============================
const x, y = 5
---
(source_file
(const_statement
(variable_declaration (identifier))
(variable_declaration (identifier) (integer_literal))))
===============================
Quote statements
===============================
quote
x = 1
y = 2
x + y
end
---
(source_file
(quote_statement
(assignment_expression (identifier) (operator) (integer_literal))
(assignment_expression (identifier) (operator) (integer_literal))
(binary_expression (identifier) (operator) (identifier))))
===============================
Try statements
===============================
try
sqrt(x)
catch
sqrt(complex(x, 0))
end
try
operate_on_file(f)
finally
close(f)
end
---
(source_file
(try_statement
(call_expression (identifier) (argument_list (identifier)))
(catch_clause
(call_expression
(identifier)
(argument_list
(call_expression (identifier) (argument_list (identifier) (integer_literal)))))))
(try_statement
(call_expression (identifier) (argument_list (identifier)))
(finally_clause
(call_expression (identifier) (argument_list (identifier))))))
===============================
Import statements
===============================
using Lib
using BigLib: thing1, thing2
import Base.show
import .A: @b
using A, B, C
---
(source_file
(import_statement (identifier))
(import_statement (selected_import (identifier) (identifier) (identifier)))
(import_statement (scoped_identifier (identifier) (identifier)))
(import_statement (selected_import (scoped_identifier (identifier)) (macro_identifier (identifier))))
(import_statement (identifier) (identifier) (identifier)))
===============================
Export statements
===============================
export a
export a, b, c
---
(source_file
(export_statement (identifier))
(export_statement (identifier) (identifier) (identifier)))