Merge branch 'master' into feature/salesforce_apex_support

pull/579/head
Rodolphe Blancho 2023-10-11 09:56:51 +07:00
commit 5256d9c98e
28 changed files with 1153 additions and 772 deletions

@ -34,8 +34,12 @@ jobs:
# Targets using cross-compilation (upload-rust-binary-action
# detects that they need cross).
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
# TODO: fix musl so it statically links during release, see
# https://github.com/Wilfred/difftastic/issues/563
#
# - target: x86_64-unknown-linux-musl
# os: ubuntu-20.04
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
- target: aarch64-apple-darwin
@ -63,4 +67,3 @@ jobs:
- uses: katyo/publish-crates@v1
with:
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

@ -1,8 +1,16 @@
## 0.52 (unreleased)
## 0.53 (unreleased)
### Command Line Interface
Added the option `--strip-cr`. This removes all carriage return
characters before diffing, which is helpful when dealing with a mix of
Windows and non-Windows flies.
## 0.52 (released 8th October 2023)
### Parsing
Added support for XML.
Added support for XML and JSONL.
### Diffing
@ -17,6 +25,16 @@ values are constructed, such as `Foo {}`).
Improved syntax highlighting for C#.
### Build
This release does not provide a prebuilt musl binary, due
to [a dynamic linking
issue](https://github.com/Wilfred/difftastic/issues/563) with binaries
in the release script.
musl remains tested in CI and supported for users, but you will need
to compile difftastic from source.
## 0.51.1 (released 25th August 2023)
Fixed an issue with GitHub actions that prevented prebuilt binaries

2
Cargo.lock generated

@ -237,7 +237,7 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
[[package]]
name = "difftastic"
version = "0.52.0"
version = "0.53.0"
dependencies = [
"assert_cmd",
"bumpalo",

@ -4,7 +4,7 @@ description = "A structural diff that understands syntax."
repository = "https://github.com/wilfred/difftastic"
homepage = "http://difftastic.wilfred.me.uk/"
license = "MIT"
version = "0.52.0"
version = "0.53.0"
authors = ["Wilfred Hughes <me@wilfred.me.uk>"]
keywords = ["diff", "syntax"]
categories = ["development-tools", "command-line-utilities", "parser-implementations"]
@ -77,6 +77,8 @@ predicates = ">= 2, <= 2.1.1"
pretty_assertions = "1.3.0"
[build-dependencies]
# TODO: enable parallel mode once MSRV hits 1.61, see discussion in
# https://github.com/rust-lang/cc-rs/pull/849
cc = "1.0.83"
rayon = "1.7.0"
version_check = "0.9.4"

@ -335,7 +335,7 @@ fn diff_file(
overrides: &[(LanguageOverride, Vec<glob::Pattern>)],
) -> DiffResult {
let (lhs_bytes, rhs_bytes) = read_files_or_die(lhs_path, rhs_path, missing_as_empty);
let (lhs_src, rhs_src) = match (guess_content(&lhs_bytes), guess_content(&rhs_bytes)) {
let (mut lhs_src, mut rhs_src) = match (guess_content(&lhs_bytes), guess_content(&rhs_bytes)) {
(ProbableFileKind::Binary, _) | (_, ProbableFileKind::Binary) => {
return DiffResult {
extra_info,
@ -353,6 +353,11 @@ fn diff_file(
(ProbableFileKind::Text(lhs_src), ProbableFileKind::Text(rhs_src)) => (lhs_src, rhs_src),
};
if diff_options.strip_cr {
lhs_src.retain(|c| c != '\r');
rhs_src.retain(|c| c != '\r');
}
diff_file_content(
display_path,
extra_info,
@ -374,7 +379,7 @@ fn diff_conflicts_file(
overrides: &[(LanguageOverride, Vec<glob::Pattern>)],
) -> DiffResult {
let bytes = read_file_or_die(path);
let src = match guess_content(&bytes) {
let mut src = match guess_content(&bytes) {
ProbableFileKind::Text(src) => src,
ProbableFileKind::Binary => {
eprintln!("error: Expected a text file with conflict markers, got a binary file.");
@ -382,6 +387,10 @@ fn diff_conflicts_file(
}
};
if diff_options.strip_cr {
src.retain(|c| c != '\r');
}
let conflict_files = match apply_conflict_markers(&src) {
Ok(cf) => cf,
Err(msg) => {

@ -68,6 +68,7 @@ pub struct DiffOptions {
pub parse_error_limit: usize,
pub check_only: bool,
pub ignore_comments: bool,
pub strip_cr: bool,
}
impl Default for DiffOptions {
@ -78,6 +79,7 @@ impl Default for DiffOptions {
parse_error_limit: DEFAULT_PARSE_ERROR_LIMIT,
check_only: false,
ignore_comments: false,
strip_cr: false,
}
}
}
@ -199,6 +201,11 @@ json: Output the results as a machine-readable JSON array with an element per fi
.env("DFT_EXIT_CODE")
.help("Set the exit code to 1 if there are syntactic changes in any files. For files where there is no detected language (e.g. unsupported language or binary files), sets the exit code if there are any byte changes.")
)
.arg(
Arg::new("strip-cr").long("strip-cr")
.env("DFT_STRIP_CR")
.help("Remove any carriage return characters before diffing. This can be helpful when dealing with files on Windows that contain CRLF, i.e. `\\r\\n`.")
)
.arg(
Arg::new("check-only").long("check-only")
.env("DFT_CHECK_ONLY")
@ -602,6 +609,8 @@ pub fn parse_args() -> Mode {
let set_exit_code = matches.is_present("exit-code");
let strip_cr = matches.is_present("strip-cr");
let check_only = matches.is_present("check-only");
let diff_options = DiffOptions {
@ -610,6 +619,7 @@ pub fn parse_args() -> Mode {
parse_error_limit,
check_only,
ignore_comments,
strip_cr,
};
let args: Vec<_> = matches.values_of_os("paths").unwrap_or_default().collect();

@ -0,0 +1,20 @@
module.exports = {
'env': {
'commonjs': true,
'es2021': true,
},
'extends': 'google',
'overrides': [
],
'parserOptions': {
'ecmaVersion': 'latest',
'sourceType': 'module',
},
'rules': {
'indent': ['error', 2, {'SwitchCase': 1}],
'max-len': [
'error',
{'code': 120, 'ignoreComments': true, 'ignoreUrls': true, 'ignoreStrings': true},
],
},
};

@ -1 +1,10 @@
/src/** linguist-vendored
/examples/* linguist-vendored
src/grammar.json linguist-generated
src/node-types.json linguist-generated
src/parser.c linguist-generated
src/grammar.json -diff
src/node-types.json -diff
src/parser.c -diff

@ -1,19 +1,24 @@
name: Build/test
name: CI
on:
push:
branches:
- "**"
- master
pull_request:
branches:
- master
jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
os: [macos-latest, ubuntu-latest, windows-2019]
os: [macos-latest, ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 14
node-version: 18
- run: npm install
- run: npm test

@ -0,0 +1,19 @@
name: Lint
on:
push:
branches:
- master
pull_request:
branches:
- "**"
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install modules
run: npm install
- name: Run ESLint
run: npm run lint

@ -0,0 +1,103 @@
name: Release
on:
workflow_run:
workflows: ["CI"]
branches:
- master
types:
- completed
jobs:
release:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Get previous commit SHA
id: get_previous_commit
run: |
LATEST_TAG=$(git describe --tags --abbrev=0)
if [[ -z "$LATEST_TAG" ]]; then
echo "No tag found. Failing..."
exit 1
fi
echo "latest_tag=${LATEST_TAG#v}" >> "$GITHUB_ENV" # Remove 'v' prefix from the tag
- name: Check if version changed and is greater than the previous
id: version_check
run: |
# Compare the current version with the version from the previous commit
PREVIOUS_NPM_VERSION=${{ env.latest_tag }}
CURRENT_NPM_VERSION=$(jq -r '.version' package.json)
CURRENT_CARGO_VERSION=$(awk -F '"' '/^version/ {print $2}' Cargo.toml)
if [[ "$CURRENT_NPM_VERSION" != "$CURRENT_CARGO_VERSION" ]]; then # Cargo.toml and package.json versions must match
echo "Mismatch: NPM version ($CURRENT_NPM_VERSION) and Cargo.toml version ($CURRENT_CARGO_VERSION)"
echo "version_changed=false" >> "$GITHUB_ENV"
else
if [[ "$PREVIOUS_NPM_VERSION" == "$CURRENT_NPM_VERSION" ]]; then
echo "version_changed=" >> "$GITHUB_ENV"
else
IFS='.' read -ra PREVIOUS_VERSION_PARTS <<< "$PREVIOUS_NPM_VERSION"
IFS='.' read -ra CURRENT_VERSION_PARTS <<< "$CURRENT_NPM_VERSION"
VERSION_CHANGED=false
for i in "${!PREVIOUS_VERSION_PARTS[@]}"; do
if [[ ${CURRENT_VERSION_PARTS[i]} -gt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
VERSION_CHANGED=true
break
elif [[ ${CURRENT_VERSION_PARTS[i]} -lt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
break
fi
done
echo "version_changed=$VERSION_CHANGED" >> "$GITHUB_ENV"
echo "current_version=${CURRENT_NPM_VERSION}" >> "$GITHUB_ENV"
fi
fi
- name: Display result
run: |
echo "Version bump detected: ${{ env.version_changed }}"
- name: Fail if version is lower
if: env.version_changed == 'false'
run: exit 1
- name: Setup Node
if: env.version_changed == 'true'
uses: actions/setup-node@v3
with:
node-version: 18
registry-url: "https://registry.npmjs.org"
- name: Publish to NPM
if: env.version_changed == 'true'
env:
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
run: npm publish
- name: Setup Rust
if: env.version_changed == 'true'
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Publish to Crates.io
if: env.version_changed == 'true'
uses: katyo/publish-crates@v2
with:
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
- name: Tag versions
if: env.version_changed == 'true'
run: |
git checkout master
git config user.name github-actions[bot]
git config user.email github-actions[bot]@users.noreply.github.com
git tag -d "v${{ env.current_version }}" || true
git push origin --delete "v${{ env.current_version }}" || true
git tag -a "v${{ env.current_version }}" -m "Version ${{ env.current_version }}"
git push origin "v${{ env.current_version }}"

@ -1,6 +1,6 @@
Cargo.lock
node_modules
build
target
*.log
package-lock.json
Cargo.lock
/target/
.build/

@ -1,4 +1,5 @@
corpus
build
script
target
/test
/examples
/build
/script
/target

@ -1,27 +1,24 @@
[package]
name = "tree-sitter-json"
description = "json grammar for the tree-sitter parsing library"
version = "0.20.0"
description = "JSON grammar for tree-sitter"
version = "0.20.1"
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
license = "MIT"
readme = "bindings/rust/README.md"
keywords = ["incremental", "parsing", "json"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-json"
edition = "2018"
license = "MIT"
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
edition = "2021"
autoexamples = false
build = "bindings/rust/build.rs"
include = [
"bindings/rust/*",
"grammar.js",
"queries/*",
"src/*",
]
include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
[lib]
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = "0.20"
tree-sitter = "~0.20.10"
[build-dependencies]
cc = "1.0"
cc = "~1.0.83"

@ -0,0 +1,114 @@
VERSION := 0.19.0
# Repository
SRC_DIR := src
PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin )
ifeq (, $(PARSER_NAME))
PARSER_NAME := $(shell basename $(PARSER_REPO_URL))
PARSER_NAME := $(subst tree-sitter-,,$(PARSER_NAME))
PARSER_NAME := $(subst .git,,$(PARSER_NAME))
endif
ifeq (, $(PARSER_URL))
PARSER_URL := $(subst :,/,$(PARSER_REPO_URL))
PARSER_URL := $(subst git@,https://,$(PARSER_URL))
PARSER_URL := $(subst .git,,$(PARSER_URL))
endif
UPPER_PARSER_NAME := $(shell echo $(PARSER_NAME) | tr a-z A-Z )
# install directory layout
PREFIX ?= /usr/local
INCLUDEDIR ?= $(PREFIX)/include
LIBDIR ?= $(PREFIX)/lib
PCLIBDIR ?= $(LIBDIR)/pkgconfig
# collect C++ sources, and link if necessary
CPPSRC := $(wildcard $(SRC_DIR)/*.cc)
ifeq (, $(CPPSRC))
ADDITIONALLIBS :=
else
ADDITIONALLIBS := -lc++
endif
# collect sources
SRC := $(wildcard $(SRC_DIR)/*.c)
SRC += $(CPPSRC)
OBJ := $(addsuffix .o,$(basename $(SRC)))
# ABI versioning
SONAME_MAJOR := 0
SONAME_MINOR := 0
CFLAGS ?= -O3 -Wall -Wextra -I$(SRC_DIR)
CXXFLAGS ?= -O3 -Wall -Wextra -I$(SRC_DIR)
override CFLAGS += -std=gnu99 -fPIC
override CXXFLAGS += -fPIC
# OS-specific bits
ifeq ($(shell uname),Darwin)
SOEXT = dylib
SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib
SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib
LINKSHARED := $(LINKSHARED)-dynamiclib -Wl,
ifneq ($(ADDITIONALLIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONALLIBS),
endif
LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/libtree-sitter-$(PARSER_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks
else
SOEXT = so
SOEXTVER_MAJOR = so.$(SONAME_MAJOR)
SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR)
LINKSHARED := $(LINKSHARED)-shared -Wl,
ifneq ($(ADDITIONALLIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONALLIBS),
endif
LINKSHARED := $(LINKSHARED)-soname,libtree-sitter-$(PARSER_NAME).so.$(SONAME_MAJOR)
endif
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
PCLIBDIR := $(PREFIX)/libdata/pkgconfig
endif
all: libtree-sitter-$(PARSER_NAME).a libtree-sitter-$(PARSER_NAME).$(SOEXTVER) bindings/c/$(PARSER_NAME).h bindings/c/tree-sitter-$(PARSER_NAME).pc
libtree-sitter-$(PARSER_NAME).a: $(OBJ)
$(AR) rcs $@ $^
libtree-sitter-$(PARSER_NAME).$(SOEXTVER): $(OBJ)
$(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@
ln -sf $@ libtree-sitter-$(PARSER_NAME).$(SOEXT)
ln -sf $@ libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR)
bindings/c/$(PARSER_NAME).h:
sed -e 's|@UPPER_PARSERNAME@|$(UPPER_PARSER_NAME)|' \
-e 's|@PARSERNAME@|$(PARSER_NAME)|' \
bindings/c/tree-sitter.h.in > $@
bindings/c/tree-sitter-$(PARSER_NAME).pc:
sed -e 's|@LIBDIR@|$(LIBDIR)|;s|@INCLUDEDIR@|$(INCLUDEDIR)|;s|@VERSION@|$(VERSION)|' \
-e 's|=$(PREFIX)|=$${prefix}|' \
-e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@ADDITIONALLIBS@|$(ADDITIONALLIBS)|' \
-e 's|@PARSERNAME@|$(PARSER_NAME)|' \
-e 's|@PARSERURL@|$(PARSER_URL)|' \
bindings/c/tree-sitter.pc.in > $@
install: all
install -d '$(DESTDIR)$(LIBDIR)'
install -m755 libtree-sitter-$(PARSER_NAME).a '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).a
install -m755 libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXTVER)
ln -sf libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR)
ln -sf libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXT)
install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
install -m644 bindings/c/$(PARSER_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/
install -d '$(DESTDIR)$(PCLIBDIR)'
install -m644 bindings/c/tree-sitter-$(PARSER_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/
clean:
rm -f $(OBJ) libtree-sitter-$(PARSER_NAME).a libtree-sitter-$(PARSER_NAME).$(SOEXT) libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR) libtree-sitter-$(PARSER_NAME).$(SOEXTVER)
rm -f bindings/c/$(PARSER_NAME).h bindings/c/tree-sitter-$(PARSER_NAME).pc
.PHONY: all install clean

@ -0,0 +1,36 @@
// swift-tools-version:5.3
import PackageDescription
let package = Package(
name: "TreeSitterJSON",
platforms: [.macOS(.v10_13), .iOS(.v11)],
products: [
.library(name: "TreeSitterJSON", targets: ["TreeSitterJSON"]),
],
dependencies: [],
targets: [
.target(name: "TreeSitterJSON",
path: ".",
exclude: [
"binding.gyp",
"bindings",
"Cargo.toml",
"corpus",
"grammar.js",
"LICENSE",
"package.json",
"README.md",
"src/grammar.json",
"src/node-types.json",
],
sources: [
"src/parser.c",
],
resources: [
.copy("queries")
],
publicHeadersPath: "bindings/swift",
cSettings: [.headerSearchPath("src")])
]
)

@ -1,6 +1,5 @@
tree-sitter-json
===========================
# tree-sitter-json
[![Build/test](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml)
[![CI](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml)
JSON grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter)

@ -0,0 +1,16 @@
#ifndef TREE_SITTER_@UPPER_PARSERNAME@_H_
#define TREE_SITTER_@UPPER_PARSERNAME@_H_
#include <tree_sitter/parser.h>
#ifdef __cplusplus
extern "C" {
#endif
extern TSLanguage *tree_sitter_@PARSERNAME@();
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_@UPPER_PARSERNAME@_H_

@ -0,0 +1,11 @@
prefix=@PREFIX@
libdir=@LIBDIR@
includedir=@INCLUDEDIR@
additionallibs=@ADDITIONALLIBS@
Name: tree-sitter-@PARSERNAME@
Description: A tree-sitter grammar for the @PARSERNAME@ programming language.
URL: @PARSERURL@
Version: @VERSION@
Libs: -L${libdir} ${additionallibs} -ltree-sitter-@PARSERNAME@
Cflags: -I${includedir}

@ -0,0 +1,37 @@
# tree-sitter-json
This crate provides a JSON grammar for the [tree-sitter][] parsing library.
To use this crate, add it to the `[dependencies]` section of your `Cargo.toml`
file. (Note that you will probably also need to depend on the
[`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful
way.)
```toml
[dependencies]
tree-sitter = "0.20.10"
tree-sitter-json = "0.20.1"
```
Typically, you will use the [language][language func] function to add this
grammar to a tree-sitter [Parser][], and then use the parser to parse some code:
```rust
let code = r#"
{
"name": "tree-sitter-json",
"description": "JSON parsing for tree-sitter",
}
"#;
let mut parser = Parser::new();
parser.set_language(tree_sitter_json::language()).expect("Error loading JSON grammar");
let parsed = parser.parse(code, None);
```
If you have any questions, please reach out to us in the [tree-sitter
discussions] page.
[language func]: https://docs.rs/tree-sitter-json/*/tree_sitter_json/fn.language.html
[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
[tree-sitter]: https://tree-sitter.github.io/
[tree-sitter crate]: https://crates.io/crates/tree-sitter
[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions

@ -0,0 +1,16 @@
#ifndef TREE_SITTER_JSON_H_
#define TREE_SITTER_JSON_H_
typedef struct TSLanguage TSLanguage;
#ifdef __cplusplus
extern "C" {
#endif
extern TSLanguage *tree_sitter_json();
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_JSON_H_

@ -94,7 +94,7 @@ Comments
{
"a": 1,
// we allow comments, because several
// commonly used tools allow comments in
// files with the extension `.json`
@ -127,3 +127,16 @@ Comments
(string
(string_content))
(number))))
===========================================
Multiple top-level objects
==========================================
{}
{}
---
(document
(object)
(object))

@ -1,3 +1,15 @@
/**
* @file JSON grammar for tree-sitter
* @author Max Brunsfeld
* @license MIT
*/
/* eslint-disable arrow-parens */
/* eslint-disable camelcase */
/* eslint-disable-next-line spaced-comment */
/// <reference types="tree-sitter-cli/dsl" />
// @ts-check
module.exports = grammar({
name: 'json',
@ -7,11 +19,11 @@ module.exports = grammar({
],
supertypes: $ => [
$._value
$._value,
],
rules: {
document: $ => $._value,
document: $ => repeat($._value),
_value: $ => choice(
$.object,
@ -20,95 +32,96 @@ module.exports = grammar({
$.string,
$.true,
$.false,
$.null
$.null,
),
object: $ => seq(
"{", commaSep($.pair), "}"
'{', commaSep($.pair), '}',
),
pair: $ => seq(
field("key", choice($.string, $.number)),
":",
field("value", $._value)
field('key', choice($.string, $.number)),
':',
field('value', $._value),
),
array: $ => seq(
"[", commaSep($._value), "]"
'[', commaSep($._value), ']',
),
string: $ => choice(
seq('"', '"'),
seq('"', $.string_content, '"')
seq('"', $.string_content, '"'),
),
string_content: $ => repeat1(choice(
token.immediate(prec(1, /[^\\"\n]+/)),
$.escape_sequence
$.escape_sequence,
)),
escape_sequence: $ => token.immediate(seq(
escape_sequence: _ => token.immediate(seq(
'\\',
/(\"|\\|\/|b|f|n|r|t|u)/
/(\"|\\|\/|b|f|n|r|t|u)/,
)),
number: $ => {
const hex_literal = seq(
choice('0x', '0X'),
/[\da-fA-F]+/
)
const decimal_digits = /\d+/
const signed_integer = seq(optional(choice('-', '+')), decimal_digits)
const exponent_part = seq(choice('e', 'E'), signed_integer)
const binary_literal = seq(choice('0b', '0B'), /[0-1]+/)
const octal_literal = seq(choice('0o', '0O'), /[0-7]+/)
number: _ => {
const decimal_digits = /\d+/;
const signed_integer = seq(optional('-'), decimal_digits);
const exponent_part = seq(choice('e', 'E'), signed_integer);
const decimal_integer_literal = seq(
optional(choice('-', '+')),
optional('-'),
choice(
'0',
seq(/[1-9]/, optional(decimal_digits))
)
)
seq(/[1-9]/, optional(decimal_digits)),
),
);
const decimal_literal = choice(
seq(decimal_integer_literal, '.', optional(decimal_digits), optional(exponent_part)),
seq('.', decimal_digits, optional(exponent_part)),
seq(decimal_integer_literal, optional(exponent_part))
)
return token(choice(
hex_literal,
decimal_literal,
binary_literal,
octal_literal
))
seq(decimal_integer_literal, optional(exponent_part)),
);
return token(decimal_literal);
},
true: $ => "true",
true: _ => 'true',
false: $ => "false",
false: _ => 'false',
null: $ => "null",
null: _ => 'null',
comment: $ => token(choice(
comment: _ => token(choice(
seq('//', /.*/),
seq(
'/*',
/[^*]*\*+([^/*][^*]*\*+)*/,
'/'
)
'/',
),
)),
}
},
});
/**
* Creates a rule to match one or more of the rules separated by a comma
*
* @param {RuleOrLiteral} rule
*
* @return {SeqRule}
*
*/
function commaSep1(rule) {
return seq(rule, repeat(seq(",", rule)))
return seq(rule, repeat(seq(',', rule)));
}
/**
* Creates a rule to optionally match one or more of the rules separated by a comma
*
* @param {RuleOrLiteral} rule
*
* @return {ChoiceRule}
*
*/
function commaSep(rule) {
return optional(commaSep1(rule))
return optional(commaSep1(rule));
}

@ -1,22 +1,26 @@
{
"name": "tree-sitter-json",
"version": "0.20.0",
"version": "0.20.1",
"description": "JSON grammar for tree-sitter",
"main": "bindings/node",
"keywords": [
"parser",
"lexer",
"json"
],
"author": "Max Brunsfeld",
"license": "MIT",
"dependencies": {
"nan": "^2.14.1"
"nan": "^2.18.0"
},
"devDependencies": {
"tree-sitter-cli": "^0.19.1"
"eslint": ">=8.50.0",
"eslint-config-google": "^0.14.0",
"tree-sitter-cli": "^0.20.8"
},
"scripts": {
"build": "tree-sitter generate && node-gyp build",
"lint": "eslint grammar.js",
"test": "tree-sitter test"
},
"tree-sitter": [
@ -24,6 +28,9 @@
"scope": "source.json",
"file-types": [
"json"
],
"highlights": [
"queries/highlights.scm"
]
}
]

@ -2,8 +2,11 @@
"name": "json",
"rules": {
"document": {
"type": "SYMBOL",
"name": "_value"
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_value"
}
},
"_value": {
"type": "CHOICE",
@ -247,105 +250,87 @@
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "0x"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "BLANK"
}
]
},
{
"type": "STRING",
"value": "0X"
}
]
},
{
"type": "PATTERN",
"value": "[\\da-fA-F]+"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
"type": "STRING",
"value": "0"
},
{
"type": "CHOICE",
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "0"
"type": "PATTERN",
"value": "[1-9]"
},
{
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "[1-9]"
"value": "\\d+"
},
{
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "BLANK"
}
]
"type": "BLANK"
}
]
}
]
}
]
},
}
]
},
{
"type": "STRING",
"value": "."
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "."
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "CHOICE",
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "PATTERN",
"value": "\\d+"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "e"
},
{
"type": "STRING",
"value": "E"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
@ -354,184 +339,97 @@
"members": [
{
"type": "STRING",
"value": "e"
"value": "-"
},
{
"type": "STRING",
"value": "E"
"type": "BLANK"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "\\d+"
}
]
"type": "PATTERN",
"value": "\\d+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "BLANK"
}
]
},
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "."
},
{
"type": "PATTERN",
"value": "\\d+"
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0"
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "e"
},
{
"type": "STRING",
"value": "E"
}
]
"type": "PATTERN",
"value": "[1-9]"
},
{
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "BLANK"
}
]
}
]
},
{
"type": "BLANK"
}
]
}
]
},
{
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0"
"value": "e"
},
{
"type": "SEQ",
"members": [
{
"type": "PATTERN",
"value": "[1-9]"
},
{
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "BLANK"
}
]
}
]
"type": "STRING",
"value": "E"
}
]
}
]
},
{
"type": "CHOICE",
"members": [
},
{
"type": "SEQ",
"members": [
@ -540,96 +438,25 @@
"members": [
{
"type": "STRING",
"value": "e"
"value": "-"
},
{
"type": "STRING",
"value": "E"
"type": "BLANK"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "\\d+"
}
]
"type": "PATTERN",
"value": "\\d+"
}
]
},
{
"type": "BLANK"
}
]
}
]
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0b"
},
{
"type": "STRING",
"value": "0B"
}
]
},
{
"type": "PATTERN",
"value": "[0-1]+"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0o"
},
{
"type": "STRING",
"value": "0O"
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "[0-7]+"
}
]
}

@ -53,8 +53,8 @@
"named": true,
"fields": {},
"children": {
"multiple": false,
"required": true,
"multiple": true,
"required": false,
"types": [
{
"type": "_value",

File diff suppressed because it is too large Load Diff

@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
@ -123,6 +122,7 @@ struct TSLanguage {
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
/*
@ -139,7 +139,8 @@ struct TSLanguage {
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
lookahead = lexer->lookahead; \
eof = lexer->eof(lexer);
#define ADVANCE(state_value) \
{ \
@ -165,7 +166,7 @@ struct TSLanguage {
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
@ -175,7 +176,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
.state = (state_value) \
} \
}}
@ -183,7 +184,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.state = (state_value), \
.repetition = true \
} \
}}