Merge commit '3fef30de8aee74600f25ec2e319b62a1a870d51e'

Update JSON parser. This allows multiple items, so JSONL is no longer
a parse error.

Fixes #528
significant_whitespace
Wilfred Hughes 2023-10-07 10:38:45 +07:00
commit ab1966b7d2
23 changed files with 1103 additions and 764 deletions

@ -2,7 +2,7 @@
### Parsing ### Parsing
Added support for XML. Added support for XML and JSONL.
### Diffing ### Diffing

@ -0,0 +1,20 @@
module.exports = {
'env': {
'commonjs': true,
'es2021': true,
},
'extends': 'google',
'overrides': [
],
'parserOptions': {
'ecmaVersion': 'latest',
'sourceType': 'module',
},
'rules': {
'indent': ['error', 2, {'SwitchCase': 1}],
'max-len': [
'error',
{'code': 120, 'ignoreComments': true, 'ignoreUrls': true, 'ignoreStrings': true},
],
},
};

@ -1 +1,10 @@
/src/** linguist-vendored /src/** linguist-vendored
/examples/* linguist-vendored
src/grammar.json linguist-generated
src/node-types.json linguist-generated
src/parser.c linguist-generated
src/grammar.json -diff
src/node-types.json -diff
src/parser.c -diff

@ -1,19 +1,24 @@
name: Build/test name: CI
on: on:
push: push:
branches: branches:
- "**" - master
pull_request:
branches:
- master
jobs: jobs:
test: test:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
strategy: strategy:
fail-fast: true fail-fast: true
matrix: matrix:
os: [macos-latest, ubuntu-latest, windows-2019] os: [macos-latest, ubuntu-latest, windows-latest]
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: actions/setup-node@v2 - uses: actions/setup-node@v3
with: with:
node-version: 14 node-version: 18
- run: npm install - run: npm install
- run: npm test - run: npm test

@ -0,0 +1,19 @@
name: Lint
on:
push:
branches:
- master
pull_request:
branches:
- "**"
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install modules
run: npm install
- name: Run ESLint
run: npm run lint

@ -0,0 +1,103 @@
name: Release
on:
workflow_run:
workflows: ["CI"]
branches:
- master
types:
- completed
jobs:
release:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Get previous commit SHA
id: get_previous_commit
run: |
LATEST_TAG=$(git describe --tags --abbrev=0)
if [[ -z "$LATEST_TAG" ]]; then
echo "No tag found. Failing..."
exit 1
fi
echo "latest_tag=${LATEST_TAG#v}" >> "$GITHUB_ENV" # Remove 'v' prefix from the tag
- name: Check if version changed and is greater than the previous
id: version_check
run: |
# Compare the current version with the version from the previous commit
PREVIOUS_NPM_VERSION=${{ env.latest_tag }}
CURRENT_NPM_VERSION=$(jq -r '.version' package.json)
CURRENT_CARGO_VERSION=$(awk -F '"' '/^version/ {print $2}' Cargo.toml)
if [[ "$CURRENT_NPM_VERSION" != "$CURRENT_CARGO_VERSION" ]]; then # Cargo.toml and package.json versions must match
echo "Mismatch: NPM version ($CURRENT_NPM_VERSION) and Cargo.toml version ($CURRENT_CARGO_VERSION)"
echo "version_changed=false" >> "$GITHUB_ENV"
else
if [[ "$PREVIOUS_NPM_VERSION" == "$CURRENT_NPM_VERSION" ]]; then
echo "version_changed=" >> "$GITHUB_ENV"
else
IFS='.' read -ra PREVIOUS_VERSION_PARTS <<< "$PREVIOUS_NPM_VERSION"
IFS='.' read -ra CURRENT_VERSION_PARTS <<< "$CURRENT_NPM_VERSION"
VERSION_CHANGED=false
for i in "${!PREVIOUS_VERSION_PARTS[@]}"; do
if [[ ${CURRENT_VERSION_PARTS[i]} -gt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
VERSION_CHANGED=true
break
elif [[ ${CURRENT_VERSION_PARTS[i]} -lt ${PREVIOUS_VERSION_PARTS[i]} ]]; then
break
fi
done
echo "version_changed=$VERSION_CHANGED" >> "$GITHUB_ENV"
echo "current_version=${CURRENT_NPM_VERSION}" >> "$GITHUB_ENV"
fi
fi
- name: Display result
run: |
echo "Version bump detected: ${{ env.version_changed }}"
- name: Fail if version is lower
if: env.version_changed == 'false'
run: exit 1
- name: Setup Node
if: env.version_changed == 'true'
uses: actions/setup-node@v3
with:
node-version: 18
registry-url: "https://registry.npmjs.org"
- name: Publish to NPM
if: env.version_changed == 'true'
env:
NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
run: npm publish
- name: Setup Rust
if: env.version_changed == 'true'
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Publish to Crates.io
if: env.version_changed == 'true'
uses: katyo/publish-crates@v2
with:
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
- name: Tag versions
if: env.version_changed == 'true'
run: |
git checkout master
git config user.name github-actions[bot]
git config user.email github-actions[bot]@users.noreply.github.com
git tag -d "v${{ env.current_version }}" || true
git push origin --delete "v${{ env.current_version }}" || true
git tag -a "v${{ env.current_version }}" -m "Version ${{ env.current_version }}"
git push origin "v${{ env.current_version }}"

@ -1,6 +1,6 @@
Cargo.lock
node_modules node_modules
build build
target
*.log
package-lock.json package-lock.json
Cargo.lock /target/
.build/

@ -1,4 +1,5 @@
corpus /test
build /examples
script /build
target /script
/target

@ -1,27 +1,24 @@
[package] [package]
name = "tree-sitter-json" name = "tree-sitter-json"
description = "json grammar for the tree-sitter parsing library" description = "JSON grammar for tree-sitter"
version = "0.20.0" version = "0.20.1"
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
license = "MIT"
readme = "bindings/rust/README.md"
keywords = ["incremental", "parsing", "json"] keywords = ["incremental", "parsing", "json"]
categories = ["parsing", "text-editors"] categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-json" repository = "https://github.com/tree-sitter/tree-sitter-json"
edition = "2018" edition = "2021"
license = "MIT" autoexamples = false
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
build = "bindings/rust/build.rs" build = "bindings/rust/build.rs"
include = [ include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"]
"bindings/rust/*",
"grammar.js",
"queries/*",
"src/*",
]
[lib] [lib]
path = "bindings/rust/lib.rs" path = "bindings/rust/lib.rs"
[dependencies] [dependencies]
tree-sitter = "0.20" tree-sitter = "~0.20.10"
[build-dependencies] [build-dependencies]
cc = "1.0" cc = "~1.0.83"

@ -0,0 +1,114 @@
VERSION := 0.19.0
# Repository
SRC_DIR := src
PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin )
ifeq (, $(PARSER_NAME))
PARSER_NAME := $(shell basename $(PARSER_REPO_URL))
PARSER_NAME := $(subst tree-sitter-,,$(PARSER_NAME))
PARSER_NAME := $(subst .git,,$(PARSER_NAME))
endif
ifeq (, $(PARSER_URL))
PARSER_URL := $(subst :,/,$(PARSER_REPO_URL))
PARSER_URL := $(subst git@,https://,$(PARSER_URL))
PARSER_URL := $(subst .git,,$(PARSER_URL))
endif
UPPER_PARSER_NAME := $(shell echo $(PARSER_NAME) | tr a-z A-Z )
# install directory layout
PREFIX ?= /usr/local
INCLUDEDIR ?= $(PREFIX)/include
LIBDIR ?= $(PREFIX)/lib
PCLIBDIR ?= $(LIBDIR)/pkgconfig
# collect C++ sources, and link if necessary
CPPSRC := $(wildcard $(SRC_DIR)/*.cc)
ifeq (, $(CPPSRC))
ADDITIONALLIBS :=
else
ADDITIONALLIBS := -lc++
endif
# collect sources
SRC := $(wildcard $(SRC_DIR)/*.c)
SRC += $(CPPSRC)
OBJ := $(addsuffix .o,$(basename $(SRC)))
# ABI versioning
SONAME_MAJOR := 0
SONAME_MINOR := 0
CFLAGS ?= -O3 -Wall -Wextra -I$(SRC_DIR)
CXXFLAGS ?= -O3 -Wall -Wextra -I$(SRC_DIR)
override CFLAGS += -std=gnu99 -fPIC
override CXXFLAGS += -fPIC
# OS-specific bits
ifeq ($(shell uname),Darwin)
SOEXT = dylib
SOEXTVER_MAJOR = $(SONAME_MAJOR).dylib
SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).dylib
LINKSHARED := $(LINKSHARED)-dynamiclib -Wl,
ifneq ($(ADDITIONALLIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONALLIBS),
endif
LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/libtree-sitter-$(PARSER_NAME).$(SONAME_MAJOR).dylib,-rpath,@executable_path/../Frameworks
else
SOEXT = so
SOEXTVER_MAJOR = so.$(SONAME_MAJOR)
SOEXTVER = so.$(SONAME_MAJOR).$(SONAME_MINOR)
LINKSHARED := $(LINKSHARED)-shared -Wl,
ifneq ($(ADDITIONALLIBS),)
LINKSHARED := $(LINKSHARED)$(ADDITIONALLIBS),
endif
LINKSHARED := $(LINKSHARED)-soname,libtree-sitter-$(PARSER_NAME).so.$(SONAME_MAJOR)
endif
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
PCLIBDIR := $(PREFIX)/libdata/pkgconfig
endif
all: libtree-sitter-$(PARSER_NAME).a libtree-sitter-$(PARSER_NAME).$(SOEXTVER) bindings/c/$(PARSER_NAME).h bindings/c/tree-sitter-$(PARSER_NAME).pc
libtree-sitter-$(PARSER_NAME).a: $(OBJ)
$(AR) rcs $@ $^
libtree-sitter-$(PARSER_NAME).$(SOEXTVER): $(OBJ)
$(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@
ln -sf $@ libtree-sitter-$(PARSER_NAME).$(SOEXT)
ln -sf $@ libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR)
bindings/c/$(PARSER_NAME).h:
sed -e 's|@UPPER_PARSERNAME@|$(UPPER_PARSER_NAME)|' \
-e 's|@PARSERNAME@|$(PARSER_NAME)|' \
bindings/c/tree-sitter.h.in > $@
bindings/c/tree-sitter-$(PARSER_NAME).pc:
sed -e 's|@LIBDIR@|$(LIBDIR)|;s|@INCLUDEDIR@|$(INCLUDEDIR)|;s|@VERSION@|$(VERSION)|' \
-e 's|=$(PREFIX)|=$${prefix}|' \
-e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@ADDITIONALLIBS@|$(ADDITIONALLIBS)|' \
-e 's|@PARSERNAME@|$(PARSER_NAME)|' \
-e 's|@PARSERURL@|$(PARSER_URL)|' \
bindings/c/tree-sitter.pc.in > $@
install: all
install -d '$(DESTDIR)$(LIBDIR)'
install -m755 libtree-sitter-$(PARSER_NAME).a '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).a
install -m755 libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXTVER)
ln -sf libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR)
ln -sf libtree-sitter-$(PARSER_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/libtree-sitter-$(PARSER_NAME).$(SOEXT)
install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
install -m644 bindings/c/$(PARSER_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/
install -d '$(DESTDIR)$(PCLIBDIR)'
install -m644 bindings/c/tree-sitter-$(PARSER_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/
clean:
rm -f $(OBJ) libtree-sitter-$(PARSER_NAME).a libtree-sitter-$(PARSER_NAME).$(SOEXT) libtree-sitter-$(PARSER_NAME).$(SOEXTVER_MAJOR) libtree-sitter-$(PARSER_NAME).$(SOEXTVER)
rm -f bindings/c/$(PARSER_NAME).h bindings/c/tree-sitter-$(PARSER_NAME).pc
.PHONY: all install clean

@ -0,0 +1,36 @@
// swift-tools-version:5.3
import PackageDescription
let package = Package(
name: "TreeSitterJSON",
platforms: [.macOS(.v10_13), .iOS(.v11)],
products: [
.library(name: "TreeSitterJSON", targets: ["TreeSitterJSON"]),
],
dependencies: [],
targets: [
.target(name: "TreeSitterJSON",
path: ".",
exclude: [
"binding.gyp",
"bindings",
"Cargo.toml",
"corpus",
"grammar.js",
"LICENSE",
"package.json",
"README.md",
"src/grammar.json",
"src/node-types.json",
],
sources: [
"src/parser.c",
],
resources: [
.copy("queries")
],
publicHeadersPath: "bindings/swift",
cSettings: [.headerSearchPath("src")])
]
)

@ -1,6 +1,5 @@
tree-sitter-json # tree-sitter-json
===========================
[![Build/test](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml) [![CI](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-json/actions/workflows/ci.yml)
JSON grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter) JSON grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter)

@ -0,0 +1,16 @@
#ifndef TREE_SITTER_@UPPER_PARSERNAME@_H_
#define TREE_SITTER_@UPPER_PARSERNAME@_H_
#include <tree_sitter/parser.h>
#ifdef __cplusplus
extern "C" {
#endif
extern TSLanguage *tree_sitter_@PARSERNAME@();
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_@UPPER_PARSERNAME@_H_

@ -0,0 +1,11 @@
prefix=@PREFIX@
libdir=@LIBDIR@
includedir=@INCLUDEDIR@
additionallibs=@ADDITIONALLIBS@
Name: tree-sitter-@PARSERNAME@
Description: A tree-sitter grammar for the @PARSERNAME@ programming language.
URL: @PARSERURL@
Version: @VERSION@
Libs: -L${libdir} ${additionallibs} -ltree-sitter-@PARSERNAME@
Cflags: -I${includedir}

@ -0,0 +1,37 @@
# tree-sitter-json
This crate provides a JSON grammar for the [tree-sitter][] parsing library.
To use this crate, add it to the `[dependencies]` section of your `Cargo.toml`
file. (Note that you will probably also need to depend on the
[`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful
way.)
```toml
[dependencies]
tree-sitter = "0.20.10"
tree-sitter-json = "0.20.1"
```
Typically, you will use the [language][language func] function to add this
grammar to a tree-sitter [Parser][], and then use the parser to parse some code:
```rust
let code = r#"
{
"name": "tree-sitter-json",
"description": "JSON parsing for tree-sitter",
}
"#;
let mut parser = Parser::new();
parser.set_language(tree_sitter_json::language()).expect("Error loading JSON grammar");
let parsed = parser.parse(code, None);
```
If you have any questions, please reach out to us in the [tree-sitter
discussions] page.
[language func]: https://docs.rs/tree-sitter-json/*/tree_sitter_json/fn.language.html
[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
[tree-sitter]: https://tree-sitter.github.io/
[tree-sitter crate]: https://crates.io/crates/tree-sitter
[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions

@ -0,0 +1,16 @@
#ifndef TREE_SITTER_JSON_H_
#define TREE_SITTER_JSON_H_
typedef struct TSLanguage TSLanguage;
#ifdef __cplusplus
extern "C" {
#endif
extern TSLanguage *tree_sitter_json();
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_JSON_H_

@ -94,7 +94,7 @@ Comments
{ {
"a": 1, "a": 1,
// we allow comments, because several // we allow comments, because several
// commonly used tools allow comments in // commonly used tools allow comments in
// files with the extension `.json` // files with the extension `.json`
@ -127,3 +127,16 @@ Comments
(string (string
(string_content)) (string_content))
(number)))) (number))))
===========================================
Multiple top-level objects
==========================================
{}
{}
---
(document
(object)
(object))

@ -1,3 +1,15 @@
/**
* @file JSON grammar for tree-sitter
* @author Max Brunsfeld
* @license MIT
*/
/* eslint-disable arrow-parens */
/* eslint-disable camelcase */
/* eslint-disable-next-line spaced-comment */
/// <reference types="tree-sitter-cli/dsl" />
// @ts-check
module.exports = grammar({ module.exports = grammar({
name: 'json', name: 'json',
@ -7,11 +19,11 @@ module.exports = grammar({
], ],
supertypes: $ => [ supertypes: $ => [
$._value $._value,
], ],
rules: { rules: {
document: $ => $._value, document: $ => repeat($._value),
_value: $ => choice( _value: $ => choice(
$.object, $.object,
@ -20,95 +32,96 @@ module.exports = grammar({
$.string, $.string,
$.true, $.true,
$.false, $.false,
$.null $.null,
), ),
object: $ => seq( object: $ => seq(
"{", commaSep($.pair), "}" '{', commaSep($.pair), '}',
), ),
pair: $ => seq( pair: $ => seq(
field("key", choice($.string, $.number)), field('key', choice($.string, $.number)),
":", ':',
field("value", $._value) field('value', $._value),
), ),
array: $ => seq( array: $ => seq(
"[", commaSep($._value), "]" '[', commaSep($._value), ']',
), ),
string: $ => choice( string: $ => choice(
seq('"', '"'), seq('"', '"'),
seq('"', $.string_content, '"') seq('"', $.string_content, '"'),
), ),
string_content: $ => repeat1(choice( string_content: $ => repeat1(choice(
token.immediate(prec(1, /[^\\"\n]+/)), token.immediate(prec(1, /[^\\"\n]+/)),
$.escape_sequence $.escape_sequence,
)), )),
escape_sequence: $ => token.immediate(seq( escape_sequence: _ => token.immediate(seq(
'\\', '\\',
/(\"|\\|\/|b|f|n|r|t|u)/ /(\"|\\|\/|b|f|n|r|t|u)/,
)), )),
number: $ => { number: _ => {
const hex_literal = seq( const decimal_digits = /\d+/;
choice('0x', '0X'), const signed_integer = seq(optional('-'), decimal_digits);
/[\da-fA-F]+/ const exponent_part = seq(choice('e', 'E'), signed_integer);
)
const decimal_digits = /\d+/
const signed_integer = seq(optional(choice('-', '+')), decimal_digits)
const exponent_part = seq(choice('e', 'E'), signed_integer)
const binary_literal = seq(choice('0b', '0B'), /[0-1]+/)
const octal_literal = seq(choice('0o', '0O'), /[0-7]+/)
const decimal_integer_literal = seq( const decimal_integer_literal = seq(
optional(choice('-', '+')), optional('-'),
choice( choice(
'0', '0',
seq(/[1-9]/, optional(decimal_digits)) seq(/[1-9]/, optional(decimal_digits)),
) ),
) );
const decimal_literal = choice( const decimal_literal = choice(
seq(decimal_integer_literal, '.', optional(decimal_digits), optional(exponent_part)), seq(decimal_integer_literal, '.', optional(decimal_digits), optional(exponent_part)),
seq('.', decimal_digits, optional(exponent_part)), seq(decimal_integer_literal, optional(exponent_part)),
seq(decimal_integer_literal, optional(exponent_part)) );
)
return token(decimal_literal);
return token(choice(
hex_literal,
decimal_literal,
binary_literal,
octal_literal
))
}, },
true: $ => "true", true: _ => 'true',
false: $ => "false", false: _ => 'false',
null: $ => "null", null: _ => 'null',
comment: $ => token(choice( comment: _ => token(choice(
seq('//', /.*/), seq('//', /.*/),
seq( seq(
'/*', '/*',
/[^*]*\*+([^/*][^*]*\*+)*/, /[^*]*\*+([^/*][^*]*\*+)*/,
'/' '/',
) ),
)), )),
} },
}); });
/**
* Creates a rule to match one or more of the rules separated by a comma
*
* @param {RuleOrLiteral} rule
*
* @return {SeqRule}
*
*/
function commaSep1(rule) { function commaSep1(rule) {
return seq(rule, repeat(seq(",", rule))) return seq(rule, repeat(seq(',', rule)));
} }
/**
* Creates a rule to optionally match one or more of the rules separated by a comma
*
* @param {RuleOrLiteral} rule
*
* @return {ChoiceRule}
*
*/
function commaSep(rule) { function commaSep(rule) {
return optional(commaSep1(rule)) return optional(commaSep1(rule));
} }

@ -1,22 +1,26 @@
{ {
"name": "tree-sitter-json", "name": "tree-sitter-json",
"version": "0.20.0", "version": "0.20.1",
"description": "JSON grammar for tree-sitter", "description": "JSON grammar for tree-sitter",
"main": "bindings/node", "main": "bindings/node",
"keywords": [ "keywords": [
"parser", "parser",
"lexer",
"json" "json"
], ],
"author": "Max Brunsfeld", "author": "Max Brunsfeld",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"nan": "^2.14.1" "nan": "^2.18.0"
}, },
"devDependencies": { "devDependencies": {
"tree-sitter-cli": "^0.19.1" "eslint": ">=8.50.0",
"eslint-config-google": "^0.14.0",
"tree-sitter-cli": "^0.20.8"
}, },
"scripts": { "scripts": {
"build": "tree-sitter generate && node-gyp build", "build": "tree-sitter generate && node-gyp build",
"lint": "eslint grammar.js",
"test": "tree-sitter test" "test": "tree-sitter test"
}, },
"tree-sitter": [ "tree-sitter": [
@ -24,6 +28,9 @@
"scope": "source.json", "scope": "source.json",
"file-types": [ "file-types": [
"json" "json"
],
"highlights": [
"queries/highlights.scm"
] ]
} }
] ]

@ -2,8 +2,11 @@
"name": "json", "name": "json",
"rules": { "rules": {
"document": { "document": {
"type": "SYMBOL", "type": "REPEAT",
"name": "_value" "content": {
"type": "SYMBOL",
"name": "_value"
}
}, },
"_value": { "_value": {
"type": "CHOICE", "type": "CHOICE",
@ -247,105 +250,87 @@
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
"type": "CHOICE", "type": "SEQ",
"members": [ "members": [
{ {
"type": "STRING", "type": "CHOICE",
"value": "0x" "members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "BLANK"
}
]
}, },
{ {
"type": "STRING", "type": "CHOICE",
"value": "0X"
}
]
},
{
"type": "PATTERN",
"value": "[\\da-fA-F]+"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "SEQ",
"members": [ "members": [
{ {
"type": "CHOICE", "type": "STRING",
"members": [ "value": "0"
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
}, },
{ {
"type": "CHOICE", "type": "SEQ",
"members": [ "members": [
{ {
"type": "STRING", "type": "PATTERN",
"value": "0" "value": "[1-9]"
}, },
{ {
"type": "SEQ", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "PATTERN", "type": "PATTERN",
"value": "[1-9]" "value": "\\d+"
}, },
{ {
"type": "CHOICE", "type": "BLANK"
"members": [
{
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "BLANK"
}
]
} }
] ]
} }
] ]
} }
] ]
}, }
]
},
{
"type": "STRING",
"value": "."
},
{
"type": "CHOICE",
"members": [
{ {
"type": "STRING", "type": "PATTERN",
"value": "." "value": "\\d+"
}, },
{ {
"type": "CHOICE", "type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [ "members": [
{ {
"type": "PATTERN", "type": "CHOICE",
"value": "\\d+" "members": [
{
"type": "STRING",
"value": "e"
},
{
"type": "STRING",
"value": "E"
}
]
}, },
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{ {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
@ -354,184 +339,97 @@
"members": [ "members": [
{ {
"type": "STRING", "type": "STRING",
"value": "e" "value": "-"
}, },
{ {
"type": "STRING", "type": "BLANK"
"value": "E"
} }
] ]
}, },
{ {
"type": "SEQ", "type": "PATTERN",
"members": [ "value": "\\d+"
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "\\d+"
}
]
} }
] ]
},
{
"type": "BLANK"
} }
] ]
},
{
"type": "BLANK"
} }
] ]
}, }
]
},
{
"type": "SEQ",
"members": [
{ {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
"type": "STRING", "type": "CHOICE",
"value": "." "members": [
}, {
{ "type": "STRING",
"type": "PATTERN", "value": "-"
"value": "\\d+" },
{
"type": "BLANK"
}
]
}, },
{ {
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
{
"type": "STRING",
"value": "0"
},
{ {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{ {
"type": "CHOICE", "type": "PATTERN",
"members": [ "value": "[1-9]"
{
"type": "STRING",
"value": "e"
},
{
"type": "STRING",
"value": "E"
}
]
}, },
{ {
"type": "SEQ", "type": "CHOICE",
"members": [ "members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{ {
"type": "PATTERN", "type": "PATTERN",
"value": "\\d+" "value": "\\d+"
},
{
"type": "BLANK"
} }
] ]
} }
] ]
},
{
"type": "BLANK"
} }
] ]
} }
] ]
}, },
{ {
"type": "SEQ", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{ {
"type": "CHOICE", "type": "CHOICE",
"members": [ "members": [
{ {
"type": "STRING", "type": "STRING",
"value": "0" "value": "e"
}, },
{ {
"type": "SEQ", "type": "STRING",
"members": [ "value": "E"
{
"type": "PATTERN",
"value": "[1-9]"
},
{
"type": "CHOICE",
"members": [
{
"type": "PATTERN",
"value": "\\d+"
},
{
"type": "BLANK"
}
]
}
]
} }
] ]
} },
]
},
{
"type": "CHOICE",
"members": [
{ {
"type": "SEQ", "type": "SEQ",
"members": [ "members": [
@ -540,96 +438,25 @@
"members": [ "members": [
{ {
"type": "STRING", "type": "STRING",
"value": "e" "value": "-"
}, },
{ {
"type": "STRING", "type": "BLANK"
"value": "E"
} }
] ]
}, },
{ {
"type": "SEQ", "type": "PATTERN",
"members": [ "value": "\\d+"
{
"type": "CHOICE",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "-"
},
{
"type": "STRING",
"value": "+"
}
]
},
{
"type": "BLANK"
}
]
},
{
"type": "PATTERN",
"value": "\\d+"
}
]
} }
] ]
},
{
"type": "BLANK"
} }
] ]
}
]
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0b"
},
{
"type": "STRING",
"value": "0B"
}
]
},
{
"type": "PATTERN",
"value": "[0-1]+"
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "0o"
}, },
{ {
"type": "STRING", "type": "BLANK"
"value": "0O"
} }
] ]
},
{
"type": "PATTERN",
"value": "[0-7]+"
} }
] ]
} }

@ -53,8 +53,8 @@
"named": true, "named": true,
"fields": {}, "fields": {},
"children": { "children": {
"multiple": false, "multiple": true,
"required": true, "required": false,
"types": [ "types": [
{ {
"type": "_value", "type": "_value",

File diff suppressed because it is too large Load Diff

@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0 #define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_ #ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol; typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId; typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage; typedef struct TSLanguage TSLanguage;
@ -123,6 +122,7 @@ struct TSLanguage {
unsigned (*serialize)(void *, char *); unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned); void (*deserialize)(void *, const char *, unsigned);
} external_scanner; } external_scanner;
const TSStateId *primary_state_ids;
}; };
/* /*
@ -139,7 +139,8 @@ struct TSLanguage {
lexer->advance(lexer, skip); \ lexer->advance(lexer, skip); \
start: \ start: \
skip = false; \ skip = false; \
lookahead = lexer->lookahead; lookahead = lexer->lookahead; \
eof = lexer->eof(lexer);
#define ADVANCE(state_value) \ #define ADVANCE(state_value) \
{ \ { \
@ -165,7 +166,7 @@ struct TSLanguage {
* Parse Table Macros * Parse Table Macros
*/ */
#define SMALL_STATE(id) id - LARGE_STATE_COUNT #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id #define STATE(id) id
@ -175,7 +176,7 @@ struct TSLanguage {
{{ \ {{ \
.shift = { \ .shift = { \
.type = TSParseActionTypeShift, \ .type = TSParseActionTypeShift, \
.state = state_value \ .state = (state_value) \
} \ } \
}} }}
@ -183,7 +184,7 @@ struct TSLanguage {
{{ \ {{ \
.shift = { \ .shift = { \
.type = TSParseActionTypeShift, \ .type = TSParseActionTypeShift, \
.state = state_value, \ .state = (state_value), \
.repetition = true \ .repetition = true \
} \ } \
}} }}