mirror of https://github.com/Wilfred/difftastic/
Start on top-level constructs, const declarations
Max Brunsfeld
commit
d29ad53a43
@ -0,0 +1 @@
|
||||
/src/** linguist-vendored
|
||||
@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
build
|
||||
*.log
|
||||
@ -0,0 +1,3 @@
|
||||
grammar_test
|
||||
build
|
||||
script
|
||||
@ -0,0 +1,14 @@
|
||||
language: node_js
|
||||
sudo: false
|
||||
node_js:
|
||||
- "node"
|
||||
compiler: clang-3.6
|
||||
env:
|
||||
- CXX=clang-3.6
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- llvm-toolchain-precise-3.6
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- clang-3.6
|
||||
@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2014 Max Brunsfeld
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@ -0,0 +1,8 @@
|
||||
tree-sitter-go
|
||||
===========================
|
||||
|
||||
[](https://travis-ci.org/tree-sitter/tree-sitter-go)
|
||||
|
||||
Golang grammar for [tree-sitter][].
|
||||
|
||||
[tree-sitter]: https://github.com/tree-sitter/tree-sitter
|
||||
@ -0,0 +1,18 @@
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"target_name": "ts_language_go_binding",
|
||||
"include_dirs": [
|
||||
"<!(node -e \"require('nan')\")",
|
||||
"src"
|
||||
],
|
||||
"sources": [
|
||||
"src/parser.c",
|
||||
"src/binding.cc"
|
||||
],
|
||||
"cflags_c": [
|
||||
"-std=c99",
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,169 @@
|
||||
const
|
||||
unicodeLetter = /[a-zA-Z]/,
|
||||
unicodeDigit = /[0-9]/,
|
||||
unicodeChar = /./,
|
||||
unicodeValue = unicodeChar,
|
||||
|
||||
hexDigit = /[0-7]/,
|
||||
octalDigit = /[0-7]/,
|
||||
decimalDigit = /[0-9]/,
|
||||
hexByteValue = seq('\\', 'x', hexDigit, hexDigit),
|
||||
octalByteValue = seq('\\', octalDigit, octalDigit, octalDigit)
|
||||
byteValue = choice(octalByteValue, hexByteValue),
|
||||
|
||||
hexLiteral = token(seq('0', choice('x', 'X'), repeat1(hexDigit))),
|
||||
octalLiteral = token(seq('0', repeat(octalDigit))),
|
||||
decimalLiteral = token(seq(/[1-9]/, repeat(decimalDigit))),
|
||||
|
||||
newline = '\n',
|
||||
letter = choice(unicodeLetter, '_'),
|
||||
|
||||
terminator = token(choice(newline, ';'))
|
||||
|
||||
const NOT_IMPLEMENTED = choice()
|
||||
|
||||
module.exports = grammar({
|
||||
name: 'go',
|
||||
|
||||
extras: $ => [
|
||||
$.comment,
|
||||
/\s/
|
||||
],
|
||||
|
||||
rules: {
|
||||
source_file: $ => seq(
|
||||
$.package_clause,
|
||||
repeat($.import_declaration),
|
||||
repeat($._top_level_declaration)
|
||||
),
|
||||
|
||||
package_clause: $ => seq(
|
||||
'package',
|
||||
$.identifier
|
||||
),
|
||||
|
||||
import_declaration: $ => seq(
|
||||
'import',
|
||||
choice(
|
||||
$.import_spec,
|
||||
seq(
|
||||
'(',
|
||||
repeat(seq(
|
||||
$.import_spec,
|
||||
terminator
|
||||
)),
|
||||
')'
|
||||
)
|
||||
)
|
||||
),
|
||||
|
||||
import_spec: $ => seq(
|
||||
optional(choice($.identifier, '.')),
|
||||
$._string_literal
|
||||
),
|
||||
|
||||
_top_level_declaration: $ => choice(
|
||||
$._declaration,
|
||||
$.function_declaration,
|
||||
$.method_declaration
|
||||
),
|
||||
|
||||
_declaration: $ => choice(
|
||||
$.const_declaration,
|
||||
$.type_declaration,
|
||||
$.var_declaration
|
||||
),
|
||||
|
||||
const_declaration: $ => seq(
|
||||
'const',
|
||||
choice(
|
||||
$.const_spec,
|
||||
seq(
|
||||
'(',
|
||||
repeat(seq($.const_spec, terminator)),
|
||||
')'
|
||||
)
|
||||
)
|
||||
),
|
||||
|
||||
const_spec: $ => seq(
|
||||
$.identifier_list,
|
||||
optional($._type),
|
||||
'=',
|
||||
$.expression_list
|
||||
),
|
||||
|
||||
function_declaration: $ => NOT_IMPLEMENTED,
|
||||
|
||||
method_declaration: $ => NOT_IMPLEMENTED,
|
||||
|
||||
type_declaration: $ => NOT_IMPLEMENTED,
|
||||
|
||||
var_declaration: $ => NOT_IMPLEMENTED,
|
||||
|
||||
identifier_list: $ => seq(
|
||||
$.identifier,
|
||||
repeat(seq(',', $.identifier))
|
||||
),
|
||||
|
||||
expression_list: $ => seq(
|
||||
$._expression,
|
||||
repeat(seq(',', $._expression))
|
||||
),
|
||||
|
||||
_type: $ => choice(
|
||||
$.identifier
|
||||
),
|
||||
|
||||
_expression: $ => choice(
|
||||
$.unary_expression,
|
||||
$.binary_expression,
|
||||
$._primary_expression
|
||||
),
|
||||
|
||||
unary_expression: $ => NOT_IMPLEMENTED,
|
||||
|
||||
binary_expression: $ => NOT_IMPLEMENTED,
|
||||
|
||||
_primary_expression: $ => choice(
|
||||
// Basic literal
|
||||
$._string_literal,
|
||||
$.int_literal,
|
||||
$.float_literal,
|
||||
|
||||
// Operand name
|
||||
$.identifier
|
||||
),
|
||||
|
||||
identifier: $ => token(seq(
|
||||
letter,
|
||||
repeat(choice(letter, unicodeDigit))
|
||||
)),
|
||||
|
||||
_string_literal: $ => choice(
|
||||
$.raw_string_literal,
|
||||
$.interpreted_string_literal
|
||||
),
|
||||
|
||||
raw_string_literal: $ => token(seq(
|
||||
'`',
|
||||
repeat(choice(unicodeChar, newline)),
|
||||
'`'
|
||||
)),
|
||||
|
||||
interpreted_string_literal: $ => token(seq(
|
||||
'"',
|
||||
repeat(choice(unicodeValue, byteValue)),
|
||||
'"'
|
||||
)),
|
||||
|
||||
int_literal: $ => choice(decimalLiteral, octalLiteral, hexLiteral),
|
||||
|
||||
float_literal: $ => NOT_IMPLEMENTED,
|
||||
|
||||
comment: $ => token(seq(
|
||||
'//',
|
||||
/.*/
|
||||
))
|
||||
}
|
||||
})
|
||||
@ -0,0 +1,68 @@
|
||||
============================================
|
||||
Single const declarations without types
|
||||
============================================
|
||||
|
||||
package main
|
||||
|
||||
const zero = 0
|
||||
const one, two = 1, 2
|
||||
const three, four, five = 3, 4, 5
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier))
|
||||
(const_declaration (const_spec
|
||||
(identifier_list (identifier))
|
||||
(expression_list (int_literal))))
|
||||
(const_declaration (const_spec
|
||||
(identifier_list (identifier) (identifier))
|
||||
(expression_list (int_literal) (int_literal))))
|
||||
(const_declaration (const_spec
|
||||
(identifier_list (identifier) (identifier) (identifier))
|
||||
(expression_list (int_literal) (int_literal) (int_literal)))))
|
||||
|
||||
============================================
|
||||
Single const declarations with types
|
||||
============================================
|
||||
|
||||
package main
|
||||
|
||||
const zero int = 0
|
||||
const one, two uint64 = 1, 2
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier))
|
||||
(const_declaration (const_spec
|
||||
(identifier_list (identifier))
|
||||
(identifier)
|
||||
(expression_list (int_literal))))
|
||||
(const_declaration (const_spec
|
||||
(identifier_list (identifier) (identifier))
|
||||
(identifier)
|
||||
(expression_list (int_literal) (int_literal)))))
|
||||
|
||||
============================================
|
||||
Grouped const declarations
|
||||
============================================
|
||||
|
||||
package main
|
||||
|
||||
const (
|
||||
zero = 0
|
||||
one = 1
|
||||
)
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier))
|
||||
(const_declaration
|
||||
(const_spec
|
||||
(identifier_list (identifier))
|
||||
(expression_list (int_literal)))
|
||||
(const_spec
|
||||
(identifier_list (identifier))
|
||||
(expression_list (int_literal)))))
|
||||
@ -0,0 +1,49 @@
|
||||
============================================
|
||||
Package clauses
|
||||
============================================
|
||||
|
||||
package main
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier)))
|
||||
|
||||
============================================
|
||||
Single import declarations
|
||||
============================================
|
||||
|
||||
package a
|
||||
|
||||
import "net/http"
|
||||
import . "some/dsl"
|
||||
import alias "some/package"
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier))
|
||||
(import_declaration (import_spec (interpreted_string_literal)))
|
||||
(import_declaration (import_spec (interpreted_string_literal)))
|
||||
(import_declaration (import_spec (identifier) (interpreted_string_literal))))
|
||||
|
||||
============================================
|
||||
Grouped import declarations
|
||||
============================================
|
||||
|
||||
package a
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
. "some/dsl"
|
||||
alias "some/package"
|
||||
)
|
||||
|
||||
----
|
||||
|
||||
(source_file
|
||||
(package_clause (identifier))
|
||||
(import_declaration
|
||||
(import_spec (interpreted_string_literal))
|
||||
(import_spec (interpreted_string_literal))
|
||||
(import_spec (identifier) (interpreted_string_literal))))
|
||||
@ -0,0 +1 @@
|
||||
module.exports = require("./build/Release/ts_language_go_binding");
|
||||
@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "tree-sitter-go",
|
||||
"version": "0.0.1",
|
||||
"description": "Go grammar for tree-sitter",
|
||||
"main": "index.js",
|
||||
"keywords": [
|
||||
"parser",
|
||||
"lexer"
|
||||
],
|
||||
"author": "Max Brunsfeld",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bindings": "1.2.x",
|
||||
"nan": "^2.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tree-sitter-compiler": ">= 0.0.45"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tree-sitter compile && node-gyp build",
|
||||
"test": "tree-sitter test"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs')
|
||||
const assert = require('assert');
|
||||
const babylon = require('babylon');
|
||||
const esprima = require('esprima');
|
||||
const treesitter = require('tree-sitter-compiler/node_modules/tree-sitter');
|
||||
const jsLanguage = require('..');
|
||||
|
||||
const ITERATION_COUNT = 50;
|
||||
|
||||
if (process.argv.length < 3) {
|
||||
console.log("Usage: script/benchmark.js <javascript-file>")
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const fileName = process.argv[2];
|
||||
const code = fs.readFileSync(fileName, 'utf8');
|
||||
|
||||
// profile("Babylon", () => {
|
||||
// let rootNode = babylon.parse(code);
|
||||
// assert(rootNode.type === 'File');
|
||||
// });
|
||||
//
|
||||
//
|
||||
// profile("Esprima", () => {
|
||||
// let rootNode = esprima.parse(code);
|
||||
// assert(rootNode.type === 'Program');
|
||||
// });
|
||||
|
||||
let document = null
|
||||
|
||||
profile("Tree-sitter", () => {
|
||||
document = new treesitter.Document()
|
||||
.setInputString(code)
|
||||
.setLanguage(jsLanguage)
|
||||
.parse();
|
||||
|
||||
assert(document.rootNode.type === 'program');
|
||||
});
|
||||
|
||||
assert(!/ERROR/.test(document.rootNode.toString()))
|
||||
|
||||
function profile (name, action) {
|
||||
console.log(name + ":")
|
||||
let durations = [];
|
||||
|
||||
for (let i = 0; i < ITERATION_COUNT; i++) {
|
||||
let startTime = Date.now();
|
||||
try {
|
||||
action();
|
||||
} catch (e) {
|
||||
console.log("FAILED", e.message);
|
||||
return
|
||||
}
|
||||
let endTime = Date.now();
|
||||
durations.push(endTime - startTime);
|
||||
}
|
||||
|
||||
durations.sort((a, b) => a - b);
|
||||
const average = durations.reduce((sum, term) => sum + term) / ITERATION_COUNT;
|
||||
const min = durations[0];
|
||||
const max = durations[durations.length - 1];
|
||||
|
||||
console.log("Average:", average, "Min:", min, "Max:", max);
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
#include "tree_sitter/runtime.h"
|
||||
#include <node.h>
|
||||
#include "nan.h"
|
||||
|
||||
using namespace v8;
|
||||
|
||||
extern "C" TSLanguage * ts_language_go();
|
||||
|
||||
namespace tree_sitter_go {
|
||||
|
||||
NAN_METHOD(New) {}
|
||||
|
||||
void Init(Handle<Object> exports, Handle<Object> module) {
|
||||
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
|
||||
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
|
||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
||||
|
||||
Local<Function> constructor = tpl->GetFunction();
|
||||
Local<Object> instance = constructor->NewInstance(0, NULL);
|
||||
Nan::SetInternalFieldPointer(instance, 0, ts_language_go());
|
||||
|
||||
instance->Set(Nan::New("name").ToLocalChecked(), Nan::New("go").ToLocalChecked());
|
||||
module->Set(Nan::New("exports").ToLocalChecked(), instance);
|
||||
}
|
||||
|
||||
NODE_MODULE(ts_language_go_binding, Init)
|
||||
|
||||
} // namespace tree_sitter_go
|
||||
@ -0,0 +1,645 @@
|
||||
{
|
||||
"name": "go",
|
||||
"rules": {
|
||||
"source_file": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "package_clause"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SYMBOL",
|
||||
"name": "import_declaration"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SYMBOL",
|
||||
"name": "_top_level_declaration"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"package_clause": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "package"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
}
|
||||
]
|
||||
},
|
||||
"import_declaration": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "import"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "import_spec"
|
||||
},
|
||||
{
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "("
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "import_spec"
|
||||
},
|
||||
{
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\n"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ";"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ")"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"import_spec": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_string_literal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"_top_level_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_declaration"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "function_declaration"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "method_declaration"
|
||||
}
|
||||
]
|
||||
},
|
||||
"_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "const_declaration"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "type_declaration"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "var_declaration"
|
||||
}
|
||||
]
|
||||
},
|
||||
"const_declaration": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "const"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "const_spec"
|
||||
},
|
||||
{
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "("
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "const_spec"
|
||||
},
|
||||
{
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\n"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ";"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ")"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"const_spec": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier_list"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_type"
|
||||
},
|
||||
{
|
||||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "="
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "expression_list"
|
||||
}
|
||||
]
|
||||
},
|
||||
"function_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"method_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"type_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"var_declaration": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"identifier_list": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ","
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"expression_list": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_expression"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ","
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_expression"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"_type": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
}
|
||||
]
|
||||
},
|
||||
"_expression": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "unary_expression"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "binary_expression"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_primary_expression"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unary_expression": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"binary_expression": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"_primary_expression": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_string_literal"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "int_literal"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "float_literal"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "identifier"
|
||||
}
|
||||
]
|
||||
},
|
||||
"identifier": {
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[a-zA-Z]"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "_"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[a-zA-Z]"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "_"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-9]"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"_string_literal": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "raw_string_literal"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "interpreted_string_literal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"raw_string_literal": {
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "`"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "."
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "`"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"interpreted_string_literal": {
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\""
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "."
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\\"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\\"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "x"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"int_literal": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[1-9]"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "PATTERN",
|
||||
"value": "[0-9]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "x"
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "X"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "REPEAT1",
|
||||
"content": {
|
||||
"type": "PATTERN",
|
||||
"value": "[0-7]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"float_literal": {
|
||||
"type": "CHOICE",
|
||||
"members": []
|
||||
},
|
||||
"comment": {
|
||||
"type": "TOKEN",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "//"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": ".*"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"extras": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "comment"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "\\s"
|
||||
}
|
||||
],
|
||||
"conflicts": []
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,203 @@
|
||||
#ifndef TREE_SITTER_PARSER_H_
|
||||
#define TREE_SITTER_PARSER_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include "tree_sitter/runtime.h"
|
||||
|
||||
#define ts_lex_state_error 0
|
||||
#define TS_DEBUG_BUFFER_SIZE 512
|
||||
|
||||
typedef struct TSTree TSTree;
|
||||
|
||||
typedef unsigned short TSStateId;
|
||||
|
||||
typedef struct {
|
||||
size_t bytes;
|
||||
size_t chars;
|
||||
size_t rows;
|
||||
size_t columns;
|
||||
} TSLength;
|
||||
|
||||
typedef struct {
|
||||
bool visible : 1;
|
||||
bool named : 1;
|
||||
bool extra : 1;
|
||||
bool structural : 1;
|
||||
} TSSymbolMetadata;
|
||||
|
||||
typedef struct TSLexer {
|
||||
void (*start_fn)(struct TSLexer *, TSStateId);
|
||||
void (*start_token_fn)(struct TSLexer *);
|
||||
bool (*advance_fn)(struct TSLexer *, TSStateId);
|
||||
TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata,
|
||||
const char *, bool fragile);
|
||||
|
||||
const char *chunk;
|
||||
size_t chunk_start;
|
||||
size_t chunk_size;
|
||||
|
||||
TSLength current_position;
|
||||
TSLength token_end_position;
|
||||
TSLength token_start_position;
|
||||
|
||||
size_t lookahead_size;
|
||||
int32_t lookahead;
|
||||
TSStateId starting_state;
|
||||
|
||||
TSInput input;
|
||||
TSDebugger debugger;
|
||||
char debug_buffer[TS_DEBUG_BUFFER_SIZE];
|
||||
} TSLexer;
|
||||
|
||||
typedef enum {
|
||||
TSParseActionTypeError,
|
||||
TSParseActionTypeShift,
|
||||
TSParseActionTypeReduce,
|
||||
TSParseActionTypeAccept,
|
||||
} TSParseActionType;
|
||||
|
||||
typedef struct {
|
||||
union {
|
||||
TSStateId to_state;
|
||||
struct {
|
||||
TSSymbol symbol;
|
||||
unsigned short child_count;
|
||||
};
|
||||
} data;
|
||||
TSParseActionType type : 3;
|
||||
bool extra : 1;
|
||||
bool fragile : 1;
|
||||
bool can_hide_split : 1;
|
||||
} TSParseAction;
|
||||
|
||||
typedef union {
|
||||
TSParseAction action;
|
||||
unsigned int count;
|
||||
} TSParseActionEntry;
|
||||
|
||||
struct TSLanguage {
|
||||
size_t symbol_count;
|
||||
const char **symbol_names;
|
||||
const TSSymbolMetadata *symbol_metadata;
|
||||
const unsigned short *parse_table;
|
||||
const TSParseActionEntry *parse_actions;
|
||||
const TSStateId *lex_states;
|
||||
TSTree *(*lex_fn)(TSLexer *, TSStateId, bool);
|
||||
};
|
||||
|
||||
/*
|
||||
* Lexer Macros
|
||||
*/
|
||||
|
||||
#define START_LEXER() \
|
||||
lexer->start_fn(lexer, state); \
|
||||
int32_t lookahead; \
|
||||
next_state: \
|
||||
lookahead = lexer->lookahead;
|
||||
|
||||
#define START_TOKEN() lexer->start_token_fn(lexer);
|
||||
|
||||
#define GO_TO_STATE(state_value) \
|
||||
{ \
|
||||
state = state_value; \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define ADVANCE(state_value) \
|
||||
{ \
|
||||
lexer->advance_fn(lexer, state_value); \
|
||||
GO_TO_STATE(state_value); \
|
||||
}
|
||||
|
||||
#define ACCEPT_FRAGILE_TOKEN(symbol) \
|
||||
return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
|
||||
ts_symbol_names[symbol], true);
|
||||
|
||||
#define ACCEPT_TOKEN(symbol) \
|
||||
return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
|
||||
ts_symbol_names[symbol], false);
|
||||
|
||||
#define LEX_ERROR() \
|
||||
if (error_mode) { \
|
||||
if (state == ts_lex_state_error) \
|
||||
lexer->advance_fn(lexer, state); \
|
||||
GO_TO_STATE(ts_lex_state_error) \
|
||||
} else { \
|
||||
ACCEPT_TOKEN(ts_builtin_sym_error) \
|
||||
}
|
||||
|
||||
/*
|
||||
* Parse Table Macros
|
||||
*/
|
||||
|
||||
enum {
|
||||
FRAGILE = 1,
|
||||
CAN_HIDE_SPLIT = 2,
|
||||
};
|
||||
|
||||
#define ERROR() \
|
||||
{ \
|
||||
{ .type = TSParseActionTypeError } \
|
||||
}
|
||||
|
||||
#define SHIFT(to_state_value, flags) \
|
||||
{ \
|
||||
{ \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.can_hide_split = (flags & CAN_HIDE_SPLIT) != 0, \
|
||||
.data = {.to_state = to_state_value } \
|
||||
} \
|
||||
}
|
||||
|
||||
#define SHIFT_EXTRA() \
|
||||
{ \
|
||||
{ .type = TSParseActionTypeShift, .extra = true } \
|
||||
}
|
||||
|
||||
#define REDUCE_EXTRA(symbol_val) \
|
||||
{ \
|
||||
{ \
|
||||
.type = TSParseActionTypeReduce, .extra = true, \
|
||||
.data = {.symbol = symbol_val, .child_count = 1 } \
|
||||
} \
|
||||
}
|
||||
|
||||
#define REDUCE(symbol_val, child_count_val, flags) \
|
||||
{ \
|
||||
{ \
|
||||
.type = TSParseActionTypeReduce, .fragile = (flags & FRAGILE) != 0, \
|
||||
.can_hide_split = (flags & CAN_HIDE_SPLIT) != 0, \
|
||||
.data = {.symbol = symbol_val, .child_count = child_count_val } \
|
||||
} \
|
||||
}
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{ \
|
||||
{ .type = TSParseActionTypeAccept } \
|
||||
}
|
||||
|
||||
#define EXPORT_LANGUAGE(language_name) \
|
||||
static TSLanguage language = { \
|
||||
.symbol_count = SYMBOL_COUNT, \
|
||||
.symbol_metadata = ts_symbol_metadata, \
|
||||
.parse_table = (const unsigned short *)ts_parse_table, \
|
||||
.parse_actions = ts_parse_actions, \
|
||||
.lex_states = ts_lex_states, \
|
||||
.symbol_names = ts_symbol_names, \
|
||||
.lex_fn = ts_lex, \
|
||||
}; \
|
||||
\
|
||||
const TSLanguage *language_name() { \
|
||||
return &language; \
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_PARSER_H_
|
||||
@ -0,0 +1,111 @@
|
||||
#ifndef TREE_SITTER_RUNTIME_H_
|
||||
#define TREE_SITTER_RUNTIME_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef unsigned short TSSymbol;
|
||||
typedef struct TSLanguage TSLanguage;
|
||||
typedef struct TSDocument TSDocument;
|
||||
|
||||
typedef enum {
|
||||
TSInputEncodingUTF8,
|
||||
TSInputEncodingUTF16,
|
||||
} TSInputEncoding;
|
||||
|
||||
typedef struct {
|
||||
void *payload;
|
||||
const char *(*read_fn)(void *payload, size_t *bytes_read);
|
||||
int (*seek_fn)(void *payload, size_t character, size_t byte);
|
||||
TSInputEncoding encoding;
|
||||
} TSInput;
|
||||
|
||||
typedef enum {
|
||||
TSDebugTypeParse,
|
||||
TSDebugTypeLex,
|
||||
} TSDebugType;
|
||||
|
||||
typedef struct {
|
||||
void *payload;
|
||||
void (*debug_fn)(void *payload, TSDebugType, const char *);
|
||||
} TSDebugger;
|
||||
|
||||
typedef struct {
|
||||
size_t position;
|
||||
size_t chars_inserted;
|
||||
size_t chars_removed;
|
||||
} TSInputEdit;
|
||||
|
||||
typedef struct {
|
||||
size_t row;
|
||||
size_t column;
|
||||
} TSPoint;
|
||||
|
||||
typedef struct {
|
||||
const void *data;
|
||||
size_t offset[3];
|
||||
} TSNode;
|
||||
|
||||
typedef struct {
|
||||
TSSymbol value;
|
||||
bool done;
|
||||
void *data;
|
||||
} TSSymbolIterator;
|
||||
|
||||
size_t ts_node_start_char(TSNode);
|
||||
size_t ts_node_start_byte(TSNode);
|
||||
TSPoint ts_node_start_point(TSNode);
|
||||
size_t ts_node_end_char(TSNode);
|
||||
size_t ts_node_end_byte(TSNode);
|
||||
TSPoint ts_node_end_point(TSNode);
|
||||
TSSymbol ts_node_symbol(TSNode);
|
||||
TSSymbolIterator ts_node_symbols(TSNode);
|
||||
void ts_symbol_iterator_next(TSSymbolIterator *);
|
||||
const char *ts_node_name(TSNode, const TSDocument *);
|
||||
char *ts_node_string(TSNode, const TSDocument *);
|
||||
bool ts_node_eq(TSNode, TSNode);
|
||||
bool ts_node_is_named(TSNode);
|
||||
bool ts_node_has_changes(TSNode);
|
||||
TSNode ts_node_parent(TSNode);
|
||||
TSNode ts_node_child(TSNode, size_t);
|
||||
TSNode ts_node_named_child(TSNode, size_t);
|
||||
size_t ts_node_child_count(TSNode);
|
||||
size_t ts_node_named_child_count(TSNode);
|
||||
TSNode ts_node_next_sibling(TSNode);
|
||||
TSNode ts_node_next_named_sibling(TSNode);
|
||||
TSNode ts_node_prev_sibling(TSNode);
|
||||
TSNode ts_node_prev_named_sibling(TSNode);
|
||||
TSNode ts_node_descendant_for_range(TSNode, size_t, size_t);
|
||||
TSNode ts_node_named_descendant_for_range(TSNode, size_t, size_t);
|
||||
|
||||
TSDocument *ts_document_make();
|
||||
void ts_document_free(TSDocument *);
|
||||
const TSLanguage *ts_document_language(TSDocument *);
|
||||
void ts_document_set_language(TSDocument *, const TSLanguage *);
|
||||
TSInput ts_document_input(TSDocument *);
|
||||
void ts_document_set_input(TSDocument *, TSInput);
|
||||
void ts_document_set_input_string(TSDocument *, const char *);
|
||||
TSDebugger ts_document_debugger(const TSDocument *);
|
||||
void ts_document_set_debugger(TSDocument *, TSDebugger);
|
||||
void ts_document_edit(TSDocument *, TSInputEdit);
|
||||
int ts_document_parse(TSDocument *);
|
||||
void ts_document_invalidate(TSDocument *);
|
||||
TSNode ts_document_root_node(const TSDocument *);
|
||||
size_t ts_document_parse_count(const TSDocument *);
|
||||
|
||||
size_t ts_language_symbol_count(const TSLanguage *);
|
||||
const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);
|
||||
|
||||
#define ts_builtin_sym_error 0
|
||||
#define ts_builtin_sym_end 1
|
||||
#define ts_builtin_sym_start 2
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_RUNTIME_H_
|
||||
Loading…
Reference in New Issue