Add 'vendor/tree-sitter-python/' from commit 'd6210ceab11e8d812d4ab59c07c81458ec6e5184'

git-subtree-dir: vendor/tree-sitter-python
git-subtree-mainline: cc93976969
git-subtree-split: d6210ceab1

vendor/tree-sitter-python/.gitattributes

@@ -0,0 +1,2 @@
+/src/** linguist-vendored
+/examples/* linguist-vendored

vendor/tree-sitter-python/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: Build/test
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - "master"
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - run: npm install
+      - run: npm test
+  test_windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - run: npm install
+      - run: npm run-script test-windows

vendor/tree-sitter-python/.gitignore

@@ -0,0 +1,7 @@
+Cargo.lock
+package-lock.json
+node_modules
+build
+*.log
+/examples/*/
+/target/

vendor/tree-sitter-python/.npmignore

@@ -0,0 +1,6 @@
+corpus
+examples
+build
+script
+target
+bindings/rust

vendor/tree-sitter-python/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "tree-sitter-python"
+description = "Python grammar for the tree-sitter parsing library"
+version = "0.19.0"
+authors = [
+    "Max Brunsfeld <maxbrunsfeld@gmail.com>",
+    "Douglas Creager <dcreager@dcreager.net>",
+]
+license = "MIT"
+readme = "bindings/rust/README.md"
+keywords = ["incremental", "parsing", "python"]
+categories = ["parsing", "text-editors"]
+repository = "https://github.com/tree-sitter/tree-sitter-python"
+edition = "2018"
+
+build = "bindings/rust/build.rs"
+include = [
+  "bindings/rust/*",
+  "grammar.js",
+  "queries/*",
+  "src/*",
+]
+
+[lib]
+path = "bindings/rust/lib.rs"
+
+[dependencies]
+tree-sitter = "0.19"
+
+[build-dependencies]
+cc = "1.0"

vendor/tree-sitter-python/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/tree-sitter-python/README.md

@@ -0,0 +1,13 @@
+tree-sitter-python
+==================
+
+[![build](https://github.com/tree-sitter/tree-sitter-python/actions/workflows/ci.yml/badge.svg)](https://github.com/tree-sitter/tree-sitter-python/actions/workflows/ci.yml)
+
+Python grammar for [tree-sitter][].
+
+[tree-sitter]: https://github.com/tree-sitter/tree-sitter
+
+#### References
+
+* [Python 2 Grammar](https://docs.python.org/2/reference/grammar.html)
+* [Python 3 Grammar](https://docs.python.org/3/reference/grammar.html)

vendor/tree-sitter-python/binding.gyp

@@ -0,0 +1,19 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_python_binding",
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")",
+        "src"
+      ],
+      "sources": [
+        "src/parser.c",
+        "bindings/node/binding.cc",
+        "src/scanner.cc"
+      ],
+      "cflags_c": [
+        "-std=c99",
+      ]
+    }
+  ]
+}

vendor/tree-sitter-python/bindings/node/binding.cc

@@ -0,0 +1,28 @@
+#include "tree_sitter/parser.h"
+#include <node.h>
+#include "nan.h"
+
+using namespace v8;
+
+extern "C" TSLanguage * tree_sitter_python();
+
+namespace {
+
+NAN_METHOD(New) {}
+
+void Init(Local<Object> exports, Local<Object> module) {
+  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
+  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
+  tpl->InstanceTemplate()->SetInternalFieldCount(1);
+
+  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
+  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
+  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_python());
+
+  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("python").ToLocalChecked());
+  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
+}
+
+NODE_MODULE(tree_sitter_python_binding, Init)
+
+}  // namespace

vendor/tree-sitter-python/bindings/node/index.js

@@ -0,0 +1,19 @@
+try {
+  module.exports = require("../../build/Release/tree_sitter_python_binding");
+} catch (error1) {
+  if (error1.code !== 'MODULE_NOT_FOUND') {
+    throw error1;
+  }
+  try {
+    module.exports = require("../../build/Debug/tree_sitter_python_binding");
+  } catch (error2) {
+    if (error2.code !== 'MODULE_NOT_FOUND') {
+      throw error2;
+    }
+    throw error1
+  }
+}
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

vendor/tree-sitter-python/bindings/rust/README.md

@@ -0,0 +1,36 @@
+# tree-sitter-python
+
+This crate provides a Python grammar for the [tree-sitter][] parsing library.
+To use this crate, add it to the `[dependencies]` section of your `Cargo.toml`
+file.  (Note that you will probably also need to depend on the
+[`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful
+way.)
+
+``` toml
+[dependencies]
+tree-sitter = "0.17"
+tree-sitter-python = "0.17"
+```
+
+Typically, you will use the [language][language func] function to add this
+grammar to a tree-sitter [Parser][], and then use the parser to parse some code:
+
+``` rust
+let code = r#"
+    def double(x):
+        return x * 2
+"#;
+let mut parser = Parser::new();
+parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
+let parsed = parser.parse(code, None);
+```
+
+If you have any questions, please reach out to us in the [tree-sitter
+discussions] page.
+
+[Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+[language func]: https://docs.rs/tree-sitter-python/*/tree_sitter_python/fn.language.html
+[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+[tree-sitter]: https://tree-sitter.github.io/
+[tree-sitter crate]: https://crates.io/crates/tree-sitter
+[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions

vendor/tree-sitter-python/bindings/rust/build.rs

@@ -0,0 +1,28 @@
+use std::path::Path;
+extern crate cc;
+
+fn main() {
+    let src_dir = Path::new("src");
+
+    let mut c_config = cc::Build::new();
+    c_config.include(&src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-trigraphs");
+    let parser_path = src_dir.join("parser.c");
+    c_config.file(&parser_path);
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+    c_config.compile("parser");
+
+    let mut cpp_config = cc::Build::new();
+    cpp_config.cpp(true);
+    cpp_config.include(&src_dir);
+    cpp_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable");
+    let scanner_path = src_dir.join("scanner.cc");
+    cpp_config.file(&scanner_path);
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    cpp_config.compile("scanner");
+}

vendor/tree-sitter-python/bindings/rust/lib.rs

@@ -0,0 +1,68 @@
+// -*- coding: utf-8 -*-
+// ------------------------------------------------------------------------------------------------
+// Copyright © 2020, tree-sitter-python authors.
+// See the LICENSE file in this repo for license details.
+// ------------------------------------------------------------------------------------------------
+
+//! This crate provides a Python grammar for the [tree-sitter][] parsing library.
+//!
+//! Typically, you will use the [language][language func] function to add this grammar to a
+//! tree-sitter [Parser][], and then use the parser to parse some code:
+//!
+//! ```
+//! use tree_sitter::Parser;
+//!
+//! let code = r#"
+//!     def double(x):
+//!         return x * 2
+//! "#;
+//! let mut parser = Parser::new();
+//! parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
+//! let parsed = parser.parse(code, None);
+//! # let parsed = parsed.unwrap();
+//! # let root = parsed.root_node();
+//! # assert!(!root.has_error());
+//! ```
+//!
+//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+//! [language func]: fn.language.html
+//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [tree-sitter]: https://tree-sitter.github.io/
+
+use tree_sitter::Language;
+
+extern "C" {
+    fn tree_sitter_python() -> Language;
+}
+
+/// Returns the tree-sitter [Language][] for this grammar.
+///
+/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+pub fn language() -> Language {
+    unsafe { tree_sitter_python() }
+}
+
+/// The source of the Python tree-sitter grammar description.
+pub const GRAMMAR: &'static str = include_str!("../../grammar.js");
+
+/// The syntax highlighting query for this language.
+pub const HIGHLIGHT_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+
+/// The content of the [`node-types.json`][] file for this grammar.
+///
+/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+
+/// The symbol tagging query for this language.
+pub const TAGGING_QUERY: &'static str = include_str!("../../queries/tags.scm");
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn can_load_grammar() {
+        let mut parser = tree_sitter::Parser::new();
+        parser
+            .set_language(super::language())
+            .expect("Error loading Python grammar");
+    }
+}

vendor/tree-sitter-python/examples/compound-statement-without-trailing-newline.py

@@ -0,0 +1,3 @@
+class Foo:
+  def bar():
+    print "hi"

vendor/tree-sitter-python/examples/crlf-line-endings.py

@@ -0,0 +1,6 @@
+print a
+
+if b:    
+    if c:
+        d
+    e

vendor/tree-sitter-python/examples/mixed-spaces-tabs.py

@@ -0,0 +1,4 @@
+def main():
+	print "hello"
+	# 1 tab = 8 spaces in Python 2
+        return

vendor/tree-sitter-python/examples/multiple-newlines.py

@@ -0,0 +1,25 @@
+def hi():
+
+
+
+    print "hi"
+
+
+def bye():
+    print "bye"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

vendor/tree-sitter-python/examples/python2-grammar-crlf.py

@@ -0,0 +1,973 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.test_support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0377, 255)
+        self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxint
+        if maxint == 2147483647:
+            self.assertEquals(-2147483647-1, -020000000000)
+            # XXX -2147483648
+            self.assert_(037777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            for s in '2147483648', '040000000000', '0x100000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxint == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
+            self.assert_(01777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            for s in '9223372036854775808', '02000000000000000000000','0x10000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxint value %r' % maxint)
+
+    def testLongIntegers(self):
+        x = 0L
+        x = 0l
+        x = 0xffffffffffffffffL
+        x = 0xffffffffffffffffl
+        x = 077777777777777777L
+        x = 077777777777777777l
+        x = 123456789012345678901234567890L
+        x = 123456789012345678901234567890l
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### 'def' NAME parameters ':' suite
+        ### parameters: '(' [varargslist] ')'
+        ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
+        ###            | ('**'|'*' '*') NAME)
+        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+        ### fpdef: NAME | '(' fplist ')'
+        ### fplist: fpdef (',' fpdef)* [',']
+        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
+        ### argument: [test '='] test   # Really [keyword '='] test
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        def f4(two, (compound, (argument, list))): pass
+        def f5((compound, first), two): pass
+        self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
+        self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
+        if sys.platform.startswith('java'):
+            self.assertEquals(f4.func_code.co_varnames,
+                   ('two', '(compound, (argument, list))', 'compound', 'argument',
+                                'list',))
+            self.assertEquals(f5.func_code.co_varnames,
+                   ('(compound, first)', 'two', 'compound', 'first'))
+        else:
+            self.assertEquals(f4.func_code.co_varnames,
+                  ('two', '.1', 'compound', 'argument',  'list'))
+            self.assertEquals(f5.func_code.co_varnames,
+                  ('.0', 'two', 'compound', 'first'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+        def v3(a, (b, c), *rest): return a, b, c, rest
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        f4(1, (2, (3, 4)))
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+        v3(1,(2,3))
+        v3(1,(2,3),4)
+        v3(1,(2,3),4,5,6,7,8,9,0)
+
+        # ceval unpacks the formal arguments into the first argcount names;
+        # thus, the names nested inside tuples must appear after these names.
+        if sys.platform.startswith('java'):
+            self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
+        else:
+            self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
+        self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+        def d31v((x)): pass
+        d31v(1)
+        def d32v((x,)): pass
+        d32v((1,))
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0L]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | print_stmt  | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testPrintStmt(self):
+        # 'print' (test ',')* [test]
+        import StringIO
+
+        # Can't test printing to real stdout without comparing output
+        # which is not available in unittest.
+        save_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+        print 1, 2, 3
+        print 1, 2, 3,
+        print
+        print 0 or 1, 0 or 1,
+        print 0 or 1
+
+        # 'print' '>>' test ','
+        print >> sys.stdout, 1, 2, 3
+        print >> sys.stdout, 1, 2, 3,
+        print >> sys.stdout
+        print >> sys.stdout, 0 or 1, 0 or 1,
+        print >> sys.stdout, 0 or 1
+
+        # test printing to an instance
+        class Gulp:
+            def write(self, msg): pass
+
+        gulp = Gulp()
+        print >> gulp, 1, 2, 3
+        print >> gulp, 1, 2, 3,
+        print >> gulp
+        print >> gulp, 0 or 1, 0 or 1,
+        print >> gulp, 0 or 1
+
+        # test print >> None
+        def driver():
+            oldstdout = sys.stdout
+            sys.stdout = Gulp()
+            try:
+                tellme(Gulp())
+                tellme()
+            finally:
+                sys.stdout = oldstdout
+
+        # we should see this once
+        def tellme(file=sys.stdout):
+            print >> file, 'hello world'
+
+        driver()
+
+        # we should not see this at all
+        def tellme(file=None):
+            print >> file, 'goodbye universe'
+
+        driver()
+
+        self.assertEqual(sys.stdout.getvalue(), '''\
+1 2 3
+1 2 3
+1 1 1
+1 2 3
+1 2 3
+1 1 1
+hello world
+''')
+        sys.stdout = save_stdout
+
+        # syntax errors
+        check_syntax_error(self, 'print ,')
+        check_syntax_error(self, 'print >> x,')
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo <> 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError, 'just testing'
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testExec(self):
+        # 'exec' expr ['in' expr [',' expr]]
+        z = None
+        del z
+        exec 'z=1+1\n'
+        if z != 2: self.fail('exec \'z=1+1\'\\n')
+        del z
+        exec 'z=1+1'
+        if z != 2: self.fail('exec \'z=1+1\'')
+        z = None
+        del z
+        import types
+        if hasattr(types, "UnicodeType"):
+            exec r"""if 1:
+            exec u'z=1+1\n'
+            if z != 2: self.fail('exec u\'z=1+1\'\\n')
+            del z
+            exec u'z=1+1'
+            if z != 2: self.fail('exec u\'z=1+1\'')"""
+        g = {}
+        exec 'z = 1' in g
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
+        g = {}
+        l = {}
+
+        import warnings
+        warnings.filterwarnings("ignore", "global statement", module="<string>")
+        exec 'global a; a = 1; b = 2' in g, l
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if l.has_key('__builtins__'): del l['__builtins__']
+        if (g, l) != ({'a':1}, {'b':2}):
+            self.fail('exec ... in g (%s), l (%s)' %(g,l))
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError, e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError, msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError), msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 <> 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort()
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
+        ### dictmaker: test ':' test (',' test ':' test)* [',']
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = `x`
+        x = `1 or 2 or 3`
+        self.assertEqual(`1,2`, '(1, 2)')
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [None < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(g.next(), [x for x in range(10)])
+        try:
+            g.next()
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            g.next()
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print x
+            return ret
+
+        self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

vendor/tree-sitter-python/examples/python2-grammar.py

@@ -0,0 +1,975 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.test_support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0377, 255)
+        self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxint
+        if maxint == 2147483647:
+            self.assertEquals(-2147483647-1, -020000000000)
+            # XXX -2147483648
+            self.assert_(037777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            for s in '2147483648', '040000000000', '0x100000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxint == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
+            self.assert_(01777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            for s in '9223372036854775808', '02000000000000000000000', \
+                     '0x10000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxint value %r' % maxint)
+
+    def testLongIntegers(self):
+        x = 0L
+        x = 0l
+        x = 0xffffffffffffffffL
+        x = 0xffffffffffffffffl
+        x = 077777777777777777L
+        x = 077777777777777777l
+        x = 123456789012345678901234567890L
+        x = 123456789012345678901234567890l
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### 'def' NAME parameters ':' suite
+        ### parameters: '(' [varargslist] ')'
+        ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
+        ###            | ('**'|'*' '*') NAME)
+        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+        ### fpdef: NAME | '(' fplist ')'
+        ### fplist: fpdef (',' fpdef)* [',']
+        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
+        ### argument: [test '='] test   # Really [keyword '='] test
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        def f4(two, (compound, (argument, list))): pass
+        def f5((compound, first), two): pass
+        self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
+        self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
+        if sys.platform.startswith('java'):
+            self.assertEquals(f4.func_code.co_varnames,
+                   ('two', '(compound, (argument, list))', 'compound', 'argument',
+                                'list',))
+            self.assertEquals(f5.func_code.co_varnames,
+                   ('(compound, first)', 'two', 'compound', 'first'))
+        else:
+            self.assertEquals(f4.func_code.co_varnames,
+                  ('two', '.1', 'compound', 'argument',  'list'))
+            self.assertEquals(f5.func_code.co_varnames,
+                  ('.0', 'two', 'compound', 'first'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+        def v3(a, (b, c), *rest): return a, b, c, rest
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        f4(1, (2, (3, 4)))
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+        v3(1,(2,3))
+        v3(1,(2,3),4)
+        v3(1,(2,3),4,5,6,7,8,9,0)
+
+        # ceval unpacks the formal arguments into the first argcount names;
+        # thus, the names nested inside tuples must appear after these names.
+        if sys.platform.startswith('java'):
+            self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
+        else:
+            self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
+        self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+        def d31v((x)): pass
+        d31v(1)
+        def d32v((x,)): pass
+        d32v((1,))
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0L]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | print_stmt  | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testPrintStmt(self):
+        # 'print' (test ',')* [test]
+        import StringIO
+
+        # Can't test printing to real stdout without comparing output
+        # which is not available in unittest.
+        save_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+        print 1, 2, 3
+        print 1, 2, 3,
+        print
+        print 0 or 1, 0 or 1,
+        print 0 or 1
+
+        # 'print' '>>' test ','
+        print >> sys.stdout, 1, 2, 3
+        print >> sys.stdout, 1, 2, 3,
+        print >> sys.stdout
+        print >> sys.stdout, 0 or 1, 0 or 1,
+        print >> sys.stdout, 0 or 1
+
+        # test printing to an instance
+        class Gulp:
+            def write(self, msg): pass
+
+        gulp = Gulp()
+        print >> gulp, 1, 2, 3
+        print >> gulp, 1, 2, 3,
+        print >> gulp
+        print >> gulp, 0 or 1, 0 or 1,
+        print >> gulp, 0 or 1
+
+        # test print >> None
+        def driver():
+            oldstdout = sys.stdout
+            sys.stdout = Gulp()
+            try:
+                tellme(Gulp())
+                tellme()
+            finally:
+                sys.stdout = oldstdout
+
+        # we should see this once
+        def tellme(file=sys.stdout):
+            print >> file, 'hello world'
+
+        driver()
+
+        # we should not see this at all
+        def tellme(file=None):
+            print >> file, 'goodbye universe'
+
+        driver()
+
+        self.assertEqual(sys.stdout.getvalue(), '''\
+1 2 3
+1 2 3
+1 1 1
+1 2 3
+1 2 3
+1 1 1
+hello world
+''')
+        sys.stdout = save_stdout
+
+        # syntax errors
+        check_syntax_error(self, 'print ,')
+        check_syntax_error(self, 'print >> x,')
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo <> 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError, 'just testing'
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testExec(self):
+        # 'exec' expr ['in' expr [',' expr]]
+        z = None
+        del z
+        exec 'z=1+1\n'
+        if z != 2: self.fail('exec \'z=1+1\'\\n')
+        del z
+        exec 'z=1+1'
+        if z != 2: self.fail('exec \'z=1+1\'')
+        z = None
+        del z
+        import types
+        if hasattr(types, "UnicodeType"):
+            exec r"""if 1:
+            exec u'z=1+1\n'
+            if z != 2: self.fail('exec u\'z=1+1\'\\n')
+            del z
+            exec u'z=1+1'
+            if z != 2: self.fail('exec u\'z=1+1\'')"""
+        g = {}
+        exec 'z = 1' in g
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
+        g = {}
+        l = {}
+
+        import warnings
+        warnings.filterwarnings("ignore", "global statement", module="<string>")
+        exec 'global a; a = 1; b = 2' in g, l
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if l.has_key('__builtins__'): del l['__builtins__']
+        if (g, l) != ({'a':1}, {'b':2}):
+            self.fail('exec ... in g (%s), l (%s)' %(g,l))
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError, e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError, msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError), msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 <> 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort()
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
+        ### dictmaker: test ':' test (',' test ':' test)* [',']
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = `x`
+        x = `1 or 2 or 3`
+        self.assertEqual(`1,2`, '(1, 2)')
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [None < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(g.next(), [x for x in range(10)])
+        try:
+            g.next()
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            g.next()
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print x
+            return ret
+
+        self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()
+

vendor/tree-sitter-python/examples/python3-grammar-crlf.py

@@ -0,0 +1,945 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(type(000), type(0))
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0o377, 255)
+        self.assertEquals(2147483647, 0o17777777777)
+        self.assertEquals(0b1001, 9)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
+            self.assertEquals(-2147483647-1, -0o20000000000)
+            # XXX -2147483648
+            self.assert_(0o37777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            self.assert_(0b1111111111111111111111111111111 > 0)
+            for s in ('2147483648', '0o40000000000', '0x100000000',
+                      '0b10000000000000000000000000000000'):
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxsize == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
+            self.assert_(0o1777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
+            for s in '9223372036854775808', '0o2000000000000000000000', \
+                     '0x10000000000000000', \
+                     '0b100000000000000000000000000000000000000000000000000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxsize value %r' % maxsize)
+
+    def testLongIntegers(self):
+        x = 0
+        x = 0xffffffffffffffff
+        x = 0Xffffffffffffffff
+        x = 0o77777777777777777
+        x = 0O77777777777777777
+        x = 123456789012345678901234567890
+        x = 0b100000000000000000000000000000000000000000000000000000000000000000000
+        x = 0B111111111111111111111111111111111111111111111111111111111111111111111
+
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testEllipsis(self):
+        x = ...
+        self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tfpdef: NAME [':' test]
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vfpdef] (',' vfpdef ['=' test])*  [',' '**' vfpdef] | '**' vfpdef)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vfpdef: NAME
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
+        self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
+        # keyword only argument tests
+        def pos0key1(*, key): return key
+        pos0key1(key=100)
+        def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
+        pos2key2(1, 2, k1=100)
+        pos2key2(1, 2, k1=100, k2=200)
+        pos2key2(1, 2, k2=100, k1=200)
+        def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
+        pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
+        pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.__annotations__, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.__annotations__, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.__annotations__, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.__annotations__, {'y': 3})
+        def f(a, b:1, c:2, d): pass
+        self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        # Check for SF Bug #1697248 - mixing decorators and a return annotation
+        def null(x): return x
+        @null
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+
+        # test closures with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+        l6 = lambda x, y, *, k=20: x+y+k
+        self.assertEquals(l6(1,2), 1+2+20)
+        self.assertEquals(l6(1,2,k=10), 1+2+10)
+
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo != 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError('just testing')
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testNonlocal(self):
+        # 'nonlocal' NAME (',' NAME)*
+        x = 0
+        y = 0
+        def f():
+            nonlocal x
+            nonlocal x, y
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError as e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr ['as' expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError as msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError) as msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
+        ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = {'one'}
+        x = {'one', 1,}
+        x = {'one', 'two', 'three'}
+        x = {2, 3, 4,}
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x): return x
+        @class_decorator
+        class G: pass
+
+    def testDictcomps(self):
+        # dictorsetmaker: ( (test ':' test (comp_for |
+        #                                   (',' test ':' test)* [','])) |
+        #                   (test (comp_for | (',' test)* [','])) )
+        nums = [1, 2, 3]
+        self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [0 < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(next(g), [x for x in range(10)])
+        try:
+            next(g)
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            next(g)
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print(x)
+            return ret
+
+        # the next line is not allowed anymore
+        #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

vendor/tree-sitter-python/examples/python3-grammar.py

@@ -0,0 +1,945 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(type(000), type(0))
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0o377, 255)
+        self.assertEquals(2147483647, 0o17777777777)
+        self.assertEquals(0b1001, 9)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
+            self.assertEquals(-2147483647-1, -0o20000000000)
+            # XXX -2147483648
+            self.assert_(0o37777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            self.assert_(0b1111111111111111111111111111111 > 0)
+            for s in ('2147483648', '0o40000000000', '0x100000000',
+                      '0b10000000000000000000000000000000'):
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxsize == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
+            self.assert_(0o1777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
+            for s in '9223372036854775808', '0o2000000000000000000000', \
+                     '0x10000000000000000', \
+                     '0b100000000000000000000000000000000000000000000000000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxsize value %r' % maxsize)
+
+    def testLongIntegers(self):
+        x = 0
+        x = 0xffffffffffffffff
+        x = 0Xffffffffffffffff
+        x = 0o77777777777777777
+        x = 0O77777777777777777
+        x = 123456789012345678901234567890
+        x = 0b100000000000000000000000000000000000000000000000000000000000000000000
+        x = 0B111111111111111111111111111111111111111111111111111111111111111111111
+
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testEllipsis(self):
+        x = ...
+        self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tfpdef: NAME [':' test]
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vfpdef] (',' vfpdef ['=' test])*  [',' '**' vfpdef] | '**' vfpdef)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vfpdef: NAME
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
+        self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
+        # keyword only argument tests
+        def pos0key1(*, key): return key
+        pos0key1(key=100)
+        def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
+        pos2key2(1, 2, k1=100)
+        pos2key2(1, 2, k1=100, k2=200)
+        pos2key2(1, 2, k2=100, k1=200)
+        def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
+        pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
+        pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.__annotations__, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.__annotations__, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.__annotations__, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.__annotations__, {'y': 3})
+        def f(a, b:1, c:2, d): pass
+        self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        # Check for SF Bug #1697248 - mixing decorators and a return annotation
+        def null(x): return x
+        @null
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+
+        # test closures with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+        l6 = lambda x, y, *, k=20: x+y+k
+        self.assertEquals(l6(1,2), 1+2+20)
+        self.assertEquals(l6(1,2,k=10), 1+2+10)
+
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo != 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError('just testing')
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testNonlocal(self):
+        # 'nonlocal' NAME (',' NAME)*
+        x = 0
+        y = 0
+        def f():
+            nonlocal x
+            nonlocal x, y
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError as e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr ['as' expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError as msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError) as msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
+        ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = {'one'}
+        x = {'one', 1,}
+        x = {'one', 'two', 'three'}
+        x = {2, 3, 4,}
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x): return x
+        @class_decorator
+        class G: pass
+
+    def testDictcomps(self):
+        # dictorsetmaker: ( (test ':' test (comp_for |
+        #                                   (',' test ':' test)* [','])) |
+        #                   (test (comp_for | (',' test)* [','])) )
+        nums = [1, 2, 3]
+        self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [0 < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(next(g), [x for x in range(10)])
+        try:
+            next(g)
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            next(g)
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print(x)
+            return ret
+
+        # the next line is not allowed anymore
+        #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = f"""
+The "quick"
+brown fo{ok()}x
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

vendor/tree-sitter-python/examples/simple-statements-without-trailing-newline.py

@@ -0,0 +1 @@
+pass; print "hi"

vendor/tree-sitter-python/examples/tabs.py

@@ -0,0 +1,32 @@
+def set_password(args):
+	password = args.password
+	while not password  :
+		password1 = getpass("" if args.quiet else "Provide password: ")
+		password_repeat = getpass("" if args.quiet else "Repeat password:  ")
+		if password1 != password_repeat:
+			print("Passwords do not match, try again")
+		elif len(password1) < 4:
+			print("Please provide at least 4 characters")
+		else:
+			password = password1
+
+	password_hash = passwd(password)
+	cfg = BaseJSONConfigManager(config_dir=jupyter_config_dir())
+	cfg.update('jupyter_notebook_config', {
+		'NotebookApp': {
+			'password': password_hash,
+		}
+	})
+	if not args.quiet:
+		print("password stored in config dir: %s" % jupyter_config_dir())
+
+def main(argv):
+	parser = argparse.ArgumentParser(argv[0])
+	subparsers = parser.add_subparsers()
+	parser_password = subparsers.add_parser('password', help='sets a password for your notebook server')
+	parser_password.add_argument("password", help="password to set, if not given, a password will be queried for (NOTE: this may not be safe)",
+			nargs="?")
+	parser_password.add_argument("--quiet", help="suppress messages", action="store_true")
+	parser_password.set_defaults(function=set_password)
+	args = parser.parse_args(argv[1:])
+	args.function(args)

vendor/tree-sitter-python/examples/trailing-whitespace.py

@@ -0,0 +1,6 @@
+print a    
+
+if b:    
+    if c:    
+        d
+    e     

vendor/tree-sitter-python/grammar.js

@@ -0,0 +1,980 @@
+const PREC = {
+  // this resolves a conflict between the usage of ':' in a lambda vs in a
+  // typed parameter. In the case of a lambda, we don't allow typed parameters.
+  lambda: -2,
+  typed_parameter: -1,
+  conditional: -1,
+
+  parenthesized_expression: 1,
+  parenthesized_list_splat: 1,
+  not: 1,
+  compare: 2,
+  or: 10,
+  and: 11,
+  bitwise_or: 12,
+  bitwise_and: 13,
+  xor: 14,
+  shift: 15,
+  plus: 16,
+  times: 17,
+  unary: 18,
+  power: 19,
+  call: 20,
+}
+
+module.exports = grammar({
+  name: 'python',
+
+  extras: $ => [
+    $.comment,
+    /[\s\f\uFEFF\u2060\u200B]|\\\r?\n/
+  ],
+
+  conflicts: $ => [
+    [$.primary_expression, $.pattern],
+    [$.primary_expression, $.list_splat_pattern],
+    [$.tuple, $.tuple_pattern],
+    [$.list, $.list_pattern],
+    [$.with_item, $._collection_elements],
+  ],
+
+  supertypes: $ => [
+    $._simple_statement,
+    $._compound_statement,
+    $.expression,
+    $.primary_expression,
+    $.pattern,
+    $.parameter,
+  ],
+
+  externals: $ => [
+    $._newline,
+    $._indent,
+    $._dedent,
+    $._string_start,
+    $._string_content,
+    $._string_end,
+  ],
+
+  inline: $ => [
+    $._simple_statement,
+    $._compound_statement,
+    $._suite,
+    $._expressions,
+    $._left_hand_side,
+    $.keyword_identifier,
+  ],
+
+  word: $ => $.identifier,
+
+  rules: {
+    module: $ => repeat($._statement),
+
+    _statement: $ => choice(
+      $._simple_statements,
+      $._compound_statement
+    ),
+
+    // Simple statements
+
+    _simple_statements: $ => seq(
+      $._simple_statement,
+      optional(repeat(seq(
+        $._semicolon,
+        $._simple_statement
+      ))),
+      optional($._semicolon),
+      $._newline
+    ),
+
+    _simple_statement: $ => choice(
+      $.future_import_statement,
+      $.import_statement,
+      $.import_from_statement,
+      $.print_statement,
+      $.assert_statement,
+      $.expression_statement,
+      $.return_statement,
+      $.delete_statement,
+      $.raise_statement,
+      $.pass_statement,
+      $.break_statement,
+      $.continue_statement,
+      $.global_statement,
+      $.nonlocal_statement,
+      $.exec_statement
+    ),
+
+    import_statement: $ => seq(
+      'import',
+      $._import_list
+    ),
+
+    import_prefix: $ => repeat1('.'),
+
+    relative_import: $ => seq(
+      $.import_prefix,
+      optional($.dotted_name)
+    ),
+
+    future_import_statement: $ => seq(
+      'from',
+      '__future__',
+      'import',
+      choice(
+        $._import_list,
+        seq('(', $._import_list, ')'),
+      )
+    ),
+
+    import_from_statement: $ => seq(
+      'from',
+      field('module_name', choice(
+        $.relative_import,
+        $.dotted_name
+      )),
+      'import',
+      choice(
+        $.wildcard_import,
+        $._import_list,
+        seq('(', $._import_list, ')')
+      )
+    ),
+
+    _import_list: $ => seq(
+      commaSep1(field('name', choice(
+        $.dotted_name,
+        $.aliased_import
+      ))),
+      optional(',')
+    ),
+
+    aliased_import: $ => seq(
+      field('name', $.dotted_name),
+      'as',
+      field('alias', $.identifier)
+    ),
+
+    wildcard_import: $ => '*',
+
+    print_statement: $ => choice(
+      prec(1, seq(
+        'print',
+        $.chevron,
+        repeat(seq(',', field('argument', $.expression))),
+        optional(','))
+      ),
+      prec(-10, seq(
+        'print',
+        commaSep1(field('argument', $.expression)),
+        optional(',')
+      ))
+    ),
+
+    chevron: $ => seq(
+      '>>',
+      $.expression
+    ),
+
+    assert_statement: $ => seq(
+      'assert',
+      commaSep1($.expression)
+    ),
+
+    expression_statement: $ => choice(
+      $.expression,
+      seq(commaSep1($.expression), optional(',')),
+      $.assignment,
+      $.augmented_assignment,
+      $.yield
+    ),
+
+    named_expression: $ => seq(
+      field('name', $.identifier),
+      ':=',
+      field('value', $.expression)
+    ),
+
+    return_statement: $ => seq(
+      'return',
+      optional($._expressions)
+    ),
+
+    delete_statement: $ => seq(
+      'del',
+      $._expressions
+    ),
+
+    _expressions: $ => choice(
+      $.expression,
+      $.expression_list
+    ),
+
+    raise_statement: $ => seq(
+      'raise',
+      optional($._expressions),
+      optional(seq('from', field('cause', $.expression)))
+    ),
+
+    pass_statement: $ => prec.left('pass'),
+    break_statement: $ => prec.left('break'),
+    continue_statement: $ => prec.left('continue'),
+
+    // Compound statements
+
+    _compound_statement: $ => choice(
+      $.if_statement,
+      $.for_statement,
+      $.while_statement,
+      $.try_statement,
+      $.with_statement,
+      $.function_definition,
+      $.class_definition,
+      $.decorated_definition
+    ),
+
+    if_statement: $ => seq(
+      'if',
+      field('condition', $.expression),
+      ':',
+      field('consequence', $._suite),
+      repeat(field('alternative', $.elif_clause)),
+      optional(field('alternative', $.else_clause))
+    ),
+
+    elif_clause: $ => seq(
+      'elif',
+      field('condition', $.expression),
+      ':',
+      field('consequence', $._suite)
+    ),
+
+    else_clause: $ => seq(
+      'else',
+      ':',
+      field('body', $._suite)
+    ),
+
+    for_statement: $ => seq(
+      optional('async'),
+      'for',
+      field('left', $._left_hand_side),
+      'in',
+      field('right', $._expressions),
+      ':',
+      field('body', $._suite),
+      field('alternative', optional($.else_clause))
+    ),
+
+    while_statement: $ => seq(
+      'while',
+      field('condition', $.expression),
+      ':',
+      field('body', $._suite),
+      optional(field('alternative', $.else_clause))
+    ),
+
+    try_statement: $ => seq(
+      'try',
+      ':',
+      field('body', $._suite),
+      choice(
+        seq(
+          repeat1($.except_clause),
+          optional($.else_clause),
+          optional($.finally_clause)
+        ),
+        $.finally_clause
+      )
+    ),
+
+    except_clause: $ => seq(
+      'except',
+      optional(seq(
+        $.expression,
+        optional(seq(
+          choice('as', ','),
+          $.expression
+        ))
+      )),
+      ':',
+      $._suite
+    ),
+
+    finally_clause: $ => seq(
+      'finally',
+      ':',
+      $._suite
+    ),
+
+    with_statement: $ => seq(
+      optional('async'),
+      'with',
+      $.with_clause,
+      ':',
+      field('body', $._suite)
+    ),
+
+    with_clause: $ => choice(
+      commaSep1($.with_item),
+      seq('(', commaSep1($.with_item), ')')
+    ),
+
+    with_item: $ => prec.dynamic(-1, seq(
+      field('value', $.expression),
+      optional(seq(
+        'as',
+        field('alias', $.pattern)
+      ))
+    )),
+
+    function_definition: $ => seq(
+      optional('async'),
+      'def',
+      field('name', $.identifier),
+      field('parameters', $.parameters),
+      optional(
+        seq(
+          '->',
+          field('return_type', $.type)
+        )
+      ),
+      ':',
+      field('body', $._suite)
+    ),
+
+    parameters: $ => seq(
+      '(',
+      optional($._parameters),
+      ')'
+    ),
+
+    lambda_parameters: $ => $._parameters,
+
+    list_splat: $ => seq(
+      '*',
+      $.expression,
+    ),
+
+    dictionary_splat: $ => seq(
+      '**',
+      $.expression
+    ),
+
+    global_statement: $ => seq(
+      'global',
+      commaSep1($.identifier)
+    ),
+
+    nonlocal_statement: $ => seq(
+      'nonlocal',
+      commaSep1($.identifier)
+    ),
+
+    exec_statement: $ => seq(
+      'exec',
+      field('code', $.string),
+      optional(
+        seq(
+          'in',
+          commaSep1($.expression)
+        )
+      )
+    ),
+
+    class_definition: $ => seq(
+      'class',
+      field('name', $.identifier),
+      field('superclasses', optional($.argument_list)),
+      ':',
+      field('body', $._suite)
+    ),
+
+    parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq(
+      '(',
+      choice(
+        alias($.parenthesized_list_splat, $.parenthesized_expression),
+        $.list_splat,
+      ),
+      ')',
+    )),
+
+    argument_list: $ => seq(
+      '(',
+      optional(commaSep1(
+        choice(
+          $.expression,
+          $.list_splat,
+          $.dictionary_splat,
+          alias($.parenthesized_list_splat, $.parenthesized_expression),
+          $.keyword_argument
+        )
+      )),
+      optional(','),
+      ')'
+    ),
+
+    decorated_definition: $ => seq(
+      repeat1($.decorator),
+      field('definition', choice(
+        $.class_definition,
+        $.function_definition
+      ))
+    ),
+
+    decorator: $ => seq(
+      '@',
+      $.primary_expression,
+      $._newline
+    ),
+
+    _suite: $ => choice(
+      alias($._simple_statements, $.block),
+      seq($._indent, $.block),
+      alias($._newline, $.block)
+    ),
+
+    block: $ => seq(
+      repeat($._statement),
+      $._dedent
+    ),
+
+    expression_list: $ => prec.right(seq(
+      $.expression,
+      choice(
+        ',',
+        seq(
+          repeat1(seq(
+            ',',
+            $.expression
+          )),
+          optional(',')
+        ),
+      )
+    )),
+
+    dotted_name: $ => sep1($.identifier, '.'),
+
+    // Patterns
+
+    _parameters: $ => seq(
+      commaSep1($.parameter),
+      optional(',')
+    ),
+
+    _patterns: $ => seq(
+      commaSep1($.pattern),
+      optional(',')
+    ),
+
+    parameter: $ => choice(
+      $.identifier,
+      $.typed_parameter,
+      $.default_parameter,
+      $.typed_default_parameter,
+      $.list_splat_pattern,
+      $.tuple_pattern,
+      alias('*', $.list_splat_pattern),
+      $.dictionary_splat_pattern
+    ),
+
+    pattern: $ => choice(
+      $.identifier,
+      $.keyword_identifier,
+      $.subscript,
+      $.attribute,
+      $.list_splat_pattern,
+      $.tuple_pattern,
+      $.list_pattern
+    ),
+
+    tuple_pattern: $ => seq(
+      '(',
+      optional($._patterns),
+      ')'
+    ),
+
+    list_pattern: $ => seq(
+      '[',
+      optional($._patterns),
+      ']'
+    ),
+
+    default_parameter: $ => seq(
+      field('name', $.identifier),
+      '=',
+      field('value', $.expression)
+    ),
+
+    typed_default_parameter: $ => prec(PREC.typed_parameter, seq(
+      field('name', $.identifier),
+      ':',
+      field('type', $.type),
+      '=',
+      field('value', $.expression)
+    )),
+
+    list_splat_pattern: $ => seq(
+      '*',
+      choice($.identifier, $.keyword_identifier, $.subscript, $.attribute)
+    ),
+
+    dictionary_splat_pattern: $ => seq(
+      '**',
+      choice($.identifier, $.keyword_identifier, $.subscript, $.attribute)
+    ),
+
+    // Expressions
+
+    _expression_within_for_in_clause: $ => choice(
+      $.expression,
+      alias($.lambda_within_for_in_clause, $.lambda)
+    ),
+
+    expression: $ => choice(
+      $.comparison_operator,
+      $.not_operator,
+      $.boolean_operator,
+      $.await,
+      $.lambda,
+      $.primary_expression,
+      $.conditional_expression,
+      $.named_expression
+    ),
+
+    primary_expression: $ => choice(
+      $.binary_operator,
+      $.identifier,
+      $.keyword_identifier,
+      $.string,
+      $.concatenated_string,
+      $.integer,
+      $.float,
+      $.true,
+      $.false,
+      $.none,
+      $.unary_operator,
+      $.attribute,
+      $.subscript,
+      $.call,
+      $.list,
+      $.list_comprehension,
+      $.dictionary,
+      $.dictionary_comprehension,
+      $.set,
+      $.set_comprehension,
+      $.tuple,
+      $.parenthesized_expression,
+      $.generator_expression,
+      $.ellipsis
+    ),
+
+    not_operator: $ => prec(PREC.not, seq(
+      'not',
+      field('argument', $.expression)
+    )),
+
+    boolean_operator: $ => choice(
+      prec.left(PREC.and, seq(
+        field('left', $.expression),
+        field('operator', 'and'),
+        field('right', $.expression)
+      )),
+      prec.left(PREC.or, seq(
+        field('left', $.expression),
+        field('operator', 'or'),
+        field('right', $.expression)
+      ))
+    ),
+
+    binary_operator: $ => {
+      const table = [
+        [prec.left, '+', PREC.plus],
+        [prec.left, '-', PREC.plus],
+        [prec.left, '*', PREC.times],
+        [prec.left, '@', PREC.times],
+        [prec.left, '/', PREC.times],
+        [prec.left, '%', PREC.times],
+        [prec.left, '//', PREC.times],
+        [prec.right, '**', PREC.power],
+        [prec.left, '|', PREC.bitwise_or],
+        [prec.left, '&', PREC.bitwise_and],
+        [prec.left, '^', PREC.xor],
+        [prec.left, '<<', PREC.shift],
+        [prec.left, '>>', PREC.shift],
+      ];
+
+      return choice(...table.map(([fn, operator, precedence]) => fn(precedence, seq(
+        field('left', $.primary_expression),
+        field('operator', operator),
+        field('right', $.primary_expression)
+      ))));
+    },
+
+    unary_operator: $ => prec(PREC.unary, seq(
+      field('operator', choice('+', '-', '~')),
+      field('argument', $.primary_expression)
+    )),
+
+    comparison_operator: $ => prec.left(PREC.compare, seq(
+      $.primary_expression,
+      repeat1(seq(
+        field('operators',
+          choice(
+            '<',
+            '<=',
+            '==',
+            '!=',
+            '>=',
+            '>',
+            '<>',
+            'in',
+            seq('not', 'in'),
+            'is',
+            seq('is', 'not')
+          )),
+          $.primary_expression
+        ))
+    )),
+
+    lambda: $ => prec(PREC.lambda, seq(
+      'lambda',
+      field('parameters', optional($.lambda_parameters)),
+      ':',
+      field('body', $.expression)
+    )),
+
+    lambda_within_for_in_clause: $ => seq(
+      'lambda',
+      field('parameters', optional($.lambda_parameters)),
+      ':',
+      field('body', $._expression_within_for_in_clause)
+    ),
+
+    assignment: $ => seq(
+      field('left', $._left_hand_side),
+      choice(
+        seq('=', field('right', $._right_hand_side)),
+        seq(':', field('type', $.type)),
+        seq(':', field('type', $.type), '=', field('right', $._right_hand_side))
+      )
+    ),
+
+    augmented_assignment: $ => seq(
+      field('left', $._left_hand_side),
+      field('operator', choice(
+        '+=', '-=', '*=', '/=', '@=', '//=', '%=', '**=',
+        '>>=', '<<=', '&=', '^=', '|='
+      )),
+      field('right', $._right_hand_side)
+    ),
+
+    _left_hand_side: $ => choice(
+      $.pattern,
+      $.pattern_list
+    ),
+
+    pattern_list: $ => seq(
+      $.pattern,
+      choice(
+        ',',
+        seq(
+          repeat1(seq(
+            ',',
+            $.pattern
+          )),
+          optional(',')
+        )
+      )
+    ),
+
+    _right_hand_side: $ => choice(
+      $.expression,
+      $.expression_list,
+      $.assignment,
+      $.augmented_assignment,
+      $.yield
+    ),
+
+    yield: $ => prec.right(seq(
+      'yield',
+      choice(
+        seq(
+          'from',
+          $.expression
+        ),
+        optional($._expressions)
+      )
+    )),
+
+    attribute: $ => prec(PREC.call, seq(
+      field('object', $.primary_expression),
+      '.',
+      field('attribute', $.identifier)
+    )),
+
+    subscript: $ => prec(PREC.call, seq(
+      field('value', $.primary_expression),
+      '[',
+      commaSep1(field('subscript', choice($.expression, $.slice))),
+      optional(','),
+      ']'
+    )),
+
+    slice: $ => seq(
+      optional($.expression),
+      ':',
+      optional($.expression),
+      optional(seq(':', optional($.expression)))
+    ),
+
+    ellipsis: $ => '...',
+
+    call: $ => prec(PREC.call, seq(
+      field('function', $.primary_expression),
+      field('arguments', choice(
+        $.generator_expression,
+        $.argument_list
+      ))
+    )),
+
+    typed_parameter: $ => prec(PREC.typed_parameter, seq(
+      choice(
+        $.identifier,
+        $.list_splat_pattern,
+        $.dictionary_splat_pattern
+      ),
+      ':',
+      field('type', $.type)
+    )),
+
+    type: $ => $.expression,
+
+    keyword_argument: $ => seq(
+      field('name', choice($.identifier, $.keyword_identifier)),
+      '=',
+      field('value', $.expression)
+    ),
+
+    // Literals
+
+    list: $ => seq(
+      '[',
+      optional($._collection_elements),
+      ']'
+    ),
+
+    set: $ => seq(
+      '{',
+      $._collection_elements,
+      '}'
+    ),
+
+    tuple: $ => seq(
+      '(',
+      optional($._collection_elements),
+      ')'
+    ),
+
+    dictionary: $ => seq(
+      '{',
+      optional(commaSep1(choice($.pair, $.dictionary_splat))),
+      optional(','),
+      '}'
+    ),
+
+    pair: $ => seq(
+      field('key', $.expression),
+      ':',
+      field('value', $.expression)
+    ),
+
+    list_comprehension: $ => seq(
+      '[',
+      field('body', $.expression),
+      $._comprehension_clauses,
+      ']'
+    ),
+
+    dictionary_comprehension: $ => seq(
+      '{',
+      field('body', $.pair),
+      $._comprehension_clauses,
+      '}'
+    ),
+
+    set_comprehension: $ => seq(
+      '{',
+      field('body', $.expression),
+      $._comprehension_clauses,
+      '}'
+    ),
+
+    generator_expression: $ => seq(
+      '(',
+      field('body', $.expression),
+      $._comprehension_clauses,
+      ')'
+    ),
+
+    _comprehension_clauses: $ => seq(
+      $.for_in_clause,
+      repeat(choice(
+        $.for_in_clause,
+        $.if_clause
+      ))
+    ),
+
+    parenthesized_expression: $ => prec(PREC.parenthesized_expression, seq(
+      '(',
+      choice($.expression, $.yield),
+      ')'
+    )),
+
+    _collection_elements: $ => seq(
+      commaSep1(choice(
+        $.expression, $.yield, $.list_splat, $.parenthesized_list_splat
+      )),
+      optional(',')
+    ),
+
+    for_in_clause: $ => prec.left(seq(
+      optional('async'),
+      'for',
+      field('left', $._left_hand_side),
+      'in',
+      field('right', commaSep1($._expression_within_for_in_clause)),
+      optional(',')
+    )),
+
+    if_clause: $ => seq(
+      'if',
+      $.expression
+    ),
+
+    conditional_expression: $ => prec.right(PREC.conditional, seq(
+      $.expression,
+      'if',
+      $.expression,
+      'else',
+      $.expression
+    )),
+
+    concatenated_string: $ => seq(
+      $.string,
+      repeat1($.string)
+    ),
+
+    string: $ => seq(
+      alias($._string_start, '"'),
+      repeat(choice($.interpolation, $.escape_sequence, $._not_escape_sequence, $._string_content)),
+      alias($._string_end, '"')
+    ),
+
+    interpolation: $ => seq(
+      '{',
+      $.expression,
+      optional($.type_conversion),
+      optional($.format_specifier),
+      '}'
+    ),
+
+    escape_sequence: $ => token(prec(1, seq(
+      '\\',
+      choice(
+        /u[a-fA-F\d]{4}/,
+        /U[a-fA-F\d]{8}/,
+        /x[a-fA-F\d]{2}/,
+        /\d{3}/,
+        /\r?\n/,
+        /['"abfrntv\\]/,
+      )
+    ))),
+
+    _not_escape_sequence: $ => '\\',
+
+    format_specifier: $ => seq(
+      ':',
+      repeat(choice(
+        token(prec(1, /[^{}\n]+/)),
+        $.format_expression
+      ))
+    ),
+
+    format_expression: $ => seq('{', $.expression, '}'),
+
+    type_conversion: $ => /![a-z]/,
+
+    integer: $ => token(choice(
+      seq(
+        choice('0x', '0X'),
+        repeat1(/_?[A-Fa-f0-9]+/),
+        optional(/[Ll]/)
+      ),
+      seq(
+        choice('0o', '0O'),
+        repeat1(/_?[0-7]+/),
+        optional(/[Ll]/)
+      ),
+      seq(
+        choice('0b', '0B'),
+        repeat1(/_?[0-1]+/),
+        optional(/[Ll]/)
+      ),
+      seq(
+        repeat1(/[0-9]+_?/),
+        choice(
+          optional(/[Ll]/), // long numbers
+          optional(/[jJ]/) // complex numbers
+        )
+      )
+    )),
+
+    float: $ => {
+      const digits = repeat1(/[0-9]+_?/);
+      const exponent = seq(/[eE][\+-]?/, digits)
+
+      return token(seq(
+        choice(
+          seq(digits, '.', optional(digits), optional(exponent)),
+          seq(optional(digits), '.', digits, optional(exponent)),
+          seq(digits, exponent)
+        ),
+        optional(choice(/[Ll]/, /[jJ]/))
+      ))
+    },
+
+    identifier: $ => /[_\p{XID_Start}][_\p{XID_Continue}]*/,
+
+    keyword_identifier: $ => prec(-3, alias(
+      choice(
+        'print',
+        'exec',
+        'async',
+        'await',
+      ),
+      $.identifier
+    )),
+
+    true: $ => 'True',
+    false: $ => 'False',
+    none: $ => 'None',
+
+    await: $ => prec(PREC.unary, seq(
+      'await',
+      $.expression
+    )),
+
+    comment: $ => token(seq('#', /.*/)),
+
+    _semicolon: $ => ';'
+  }
+})
+
+function commaSep1 (rule) {
+  return sep1(rule, ',')
+}
+
+function sep1 (rule, separator) {
+  return seq(rule, repeat(seq(separator, rule)))
+}

vendor/tree-sitter-python/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "tree-sitter-python",
+  "version": "0.19.0",
+  "description": "Python grammar for tree-sitter",
+  "main": "bindings/node",
+  "keywords": [
+    "parser",
+    "lexer"
+  ],
+  "author": "Max Brunsfeld",
+  "license": "MIT",
+  "dependencies": {
+    "nan": "^2.14.0"
+  },
+  "devDependencies": {
+    "tree-sitter-cli": "^0.19.3"
+  },
+  "scripts": {
+    "build": "tree-sitter generate && node-gyp build",
+    "test": "tree-sitter test && script/parse-examples",
+    "test-windows": "tree-sitter test"
+  },
+  "repository": "https://github.com/tree-sitter/tree-sitter-python",
+  "tree-sitter": [
+    {
+      "scope": "source.python",
+      "file-types": [
+        "py"
+      ]
+    }
+  ]
+}

vendor/tree-sitter-python/queries/highlights.scm

@@ -0,0 +1,124 @@
+; Identifier naming conventions
+
+((identifier) @constructor
+ (#match? @constructor "^[A-Z]"))
+
+((identifier) @constant
+ (#match? @constant "^[A-Z][A-Z_]*$"))
+
+; Builtin functions
+
+((call
+  function: (identifier) @function.builtin)
+ (#match?
+   @function.builtin
+   "^(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dict|dir|divmod|enumerate|eval|exec|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip|__import__)$"))
+
+; Function calls
+
+(decorator) @function
+
+(call
+  function: (attribute attribute: (identifier) @function.method))
+(call
+  function: (identifier) @function)
+
+; Function definitions
+
+(function_definition
+  name: (identifier) @function)
+
+(identifier) @variable
+(attribute attribute: (identifier) @property)
+(type (identifier) @type)
+
+; Literals
+
+[
+  (none)
+  (true)
+  (false)
+] @constant.builtin
+
+[
+  (integer)
+  (float)
+] @number
+
+(comment) @comment
+(string) @string
+(escape_sequence) @escape
+
+(interpolation
+  "{" @punctuation.special
+  "}" @punctuation.special) @embedded
+
+[
+  "-"
+  "-="
+  "!="
+  "*"
+  "**"
+  "**="
+  "*="
+  "/"
+  "//"
+  "//="
+  "/="
+  "&"
+  "%"
+  "%="
+  "^"
+  "+"
+  "->"
+  "+="
+  "<"
+  "<<"
+  "<="
+  "<>"
+  "="
+  ":="
+  "=="
+  ">"
+  ">="
+  ">>"
+  "|"
+  "~"
+  "and"
+  "in"
+  "is"
+  "not"
+  "or"
+] @operator
+
+[
+  "as"
+  "assert"
+  "async"
+  "await"
+  "break"
+  "class"
+  "continue"
+  "def"
+  "del"
+  "elif"
+  "else"
+  "except"
+  "exec"
+  "finally"
+  "for"
+  "from"
+  "global"
+  "if"
+  "import"
+  "lambda"
+  "nonlocal"
+  "pass"
+  "print"
+  "raise"
+  "return"
+  "try"
+  "while"
+  "with"
+  "yield"
+] @keyword

vendor/tree-sitter-python/queries/tags.scm

@@ -0,0 +1,12 @@
+(class_definition
+  name: (identifier) @name) @definition.class
+
+(function_definition
+  name: (identifier) @name) @definition.function
+
+(call
+  function: [
+      (identifier) @name
+      (attribute
+        attribute: (identifier) @name)
+  ]) @reference.call

vendor/tree-sitter-python/script/parse-examples

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+function checkout() {
+  repo=$1; url=$2; sha=$3
+
+  if [ ! -d "$repo" ]; then
+    git clone "https://github.com/$url" "$repo"
+  fi
+
+  pushd "$repo"
+  git fetch && git reset --hard "$sha"
+  popd
+}
+
+checkout examples/numpy numpy/numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
+checkout examples/django django/django 01974d7f7549b2dca2a729c3c1a1ea7d4585eb3a
+checkout examples/flask pallets/flask de464c03e134127140e5622e230790806a133ff9
+
+known_failures="$(cat script/known_failures.txt)"
+
+tree-sitter parse -q \
+  'examples/**/*.py' \
+  $(for file in $known_failures; do echo "!${file}"; done)
+
+example_count=$(find examples -name '*.py' | wc -l)
+failure_count=$(wc -w <<< "$known_failures")
+success_count=$(( $example_count - $failure_count ))
+success_percent=$(bc -l <<< "100*${success_count}/${example_count}")
+
+printf \
+  "Successfully parsed %d of %d example files (%.1f%%)\n" \
+  $success_count $example_count $success_percent

vendor/tree-sitter-python/src/scanner.cc

@@ -0,0 +1,395 @@
+#include <tree_sitter/parser.h>
+#include <vector>
+#include <cwctype>
+#include <cstring>
+#include <cassert>
+#include <stdio.h>
+namespace {
+
+using std::vector;
+using std::iswspace;
+using std::memcpy;
+
+enum TokenType {
+  NEWLINE,
+  INDENT,
+  DEDENT,
+  STRING_START,
+  STRING_CONTENT,
+  STRING_END,
+};
+
+struct Delimiter {
+  enum {
+    SingleQuote = 1 << 0,
+    DoubleQuote = 1 << 1,
+    BackQuote = 1 << 2,
+    Raw = 1 << 3,
+    Format = 1 << 4,
+    Triple = 1 << 5,
+    Bytes = 1 << 6,
+  };
+
+  Delimiter() : flags(0) {}
+
+  bool is_format() const {
+    return flags & Format;
+  }
+
+  bool is_raw() const {
+    return flags & Raw;
+  }
+
+  bool is_triple() const {
+    return flags & Triple;
+  }
+
+  bool is_bytes() const {
+    return flags & Bytes;
+  }
+
+  int32_t end_character() const {
+    if (flags & SingleQuote) return '\'';
+    if (flags & DoubleQuote) return '"';
+    if (flags & BackQuote) return '`';
+    return 0;
+  }
+
+  void set_format() {
+    flags |= Format;
+  }
+
+  void set_raw() {
+    flags |= Raw;
+  }
+
+  void set_triple() {
+    flags |= Triple;
+  }
+
+  void set_bytes() {
+    flags |= Bytes;
+  }
+
+  void set_end_character(int32_t character) {
+    switch (character) {
+      case '\'':
+        flags |= SingleQuote;
+        break;
+      case '"':
+        flags |= DoubleQuote;
+        break;
+      case '`':
+        flags |= BackQuote;
+        break;
+      default:
+        assert(false);
+    }
+  }
+
+  char flags;
+};
+
+struct Scanner {
+  Scanner() {
+    assert(sizeof(Delimiter) == sizeof(char));
+    deserialize(NULL, 0);
+  }
+
+  unsigned serialize(char *buffer) {
+    size_t i = 0;
+
+    size_t delimiter_count = delimiter_stack.size();
+    if (delimiter_count > UINT8_MAX) delimiter_count = UINT8_MAX;
+    buffer[i++] = delimiter_count;
+
+    if (delimiter_count > 0) {
+      memcpy(&buffer[i], delimiter_stack.data(), delimiter_count);
+    }
+    i += delimiter_count;
+
+    vector<uint16_t>::iterator
+      iter = indent_length_stack.begin() + 1,
+      end = indent_length_stack.end();
+
+    for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) {
+      buffer[i++] = *iter;
+    }
+
+    return i;
+  }
+
+  void deserialize(const char *buffer, unsigned length) {
+    delimiter_stack.clear();
+    indent_length_stack.clear();
+    indent_length_stack.push_back(0);
+
+    if (length > 0) {
+      size_t i = 0;
+
+      size_t delimiter_count = (uint8_t)buffer[i++];
+      delimiter_stack.resize(delimiter_count);
+      if (delimiter_count > 0) {
+        memcpy(delimiter_stack.data(), &buffer[i], delimiter_count);
+      }
+      i += delimiter_count;
+
+      for (; i < length; i++) {
+        indent_length_stack.push_back(buffer[i]);
+      }
+    }
+  }
+
+  void advance(TSLexer *lexer) {
+    lexer->advance(lexer, false);
+  }
+
+  void skip(TSLexer *lexer) {
+    lexer->advance(lexer, true);
+  }
+
+  bool scan(TSLexer *lexer, const bool *valid_symbols) {
+    if (valid_symbols[STRING_CONTENT] && !valid_symbols[INDENT] && !delimiter_stack.empty()) {
+      Delimiter delimiter = delimiter_stack.back();
+      int32_t end_character = delimiter.end_character();
+      bool has_content = false;
+      while (lexer->lookahead) {
+        if (lexer->lookahead == '{' && delimiter.is_format()) {
+          lexer->mark_end(lexer);
+          lexer->advance(lexer, false);
+          if (lexer->lookahead == '{') {
+            lexer->advance(lexer, false);
+          } else {
+            lexer->result_symbol = STRING_CONTENT;
+            return has_content;
+          }
+        } else if (lexer->lookahead == '\\') {
+          if (delimiter.is_raw()) {
+            lexer->advance(lexer, false);
+          } else if (delimiter.is_bytes()) {
+              lexer->mark_end(lexer);
+              lexer->advance(lexer, false);
+              if (lexer->lookahead == 'N' || lexer->lookahead == 'u' || lexer->lookahead == 'U') {
+                // In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are not escape sequences
+                // https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
+                lexer->advance(lexer, false);
+              } else {
+                  lexer->result_symbol = STRING_CONTENT;
+                  return has_content;
+              }
+          } else {
+            lexer->mark_end(lexer);
+            lexer->result_symbol = STRING_CONTENT;
+            return has_content;
+          }
+        } else if (lexer->lookahead == end_character) {
+          if (delimiter.is_triple()) {
+            lexer->mark_end(lexer);
+            lexer->advance(lexer, false);
+            if (lexer->lookahead == end_character) {
+              lexer->advance(lexer, false);
+              if (lexer->lookahead == end_character) {
+                if (has_content) {
+                  lexer->result_symbol = STRING_CONTENT;
+                } else {
+                  lexer->advance(lexer, false);
+                  lexer->mark_end(lexer);
+                  delimiter_stack.pop_back();
+                  lexer->result_symbol = STRING_END;
+                }
+                return true;
+              }
+            }
+          } else {
+            if (has_content) {
+              lexer->result_symbol = STRING_CONTENT;
+            } else {
+              lexer->advance(lexer, false);
+              delimiter_stack.pop_back();
+              lexer->result_symbol = STRING_END;
+            }
+            lexer->mark_end(lexer);
+            return true;
+          }
+        } else if (lexer->lookahead == '\n' && has_content && !delimiter.is_triple()) {
+          return false;
+        }
+        advance(lexer);
+        has_content = true;
+      }
+    }
+
+    lexer->mark_end(lexer);
+
+    bool found_end_of_line = false;
+    uint32_t indent_length = 0;
+    int32_t first_comment_indent_length = -1;
+    for (;;) {
+      if (lexer->lookahead == '\n') {
+        found_end_of_line = true;
+        indent_length = 0;
+        skip(lexer);
+      } else if (lexer->lookahead == ' ') {
+        indent_length++;
+        skip(lexer);
+      } else if (lexer->lookahead == '\r') {
+        indent_length = 0;
+        skip(lexer);
+      } else if (lexer->lookahead == '\t') {
+        indent_length += 8;
+        skip(lexer);
+      } else if (lexer->lookahead == '#') {
+        if (first_comment_indent_length == -1) {
+          first_comment_indent_length = (int32_t)indent_length;
+        }
+        while (lexer->lookahead && lexer->lookahead != '\n') {
+          skip(lexer);
+        }
+        skip(lexer);
+        indent_length = 0;
+      } else if (lexer->lookahead == '\\') {
+        skip(lexer);
+        if (iswspace(lexer->lookahead)) {
+          skip(lexer);
+        } else {
+          return false;
+        }
+      } else if (lexer->lookahead == '\f') {
+        indent_length = 0;
+        skip(lexer);
+      } else if (lexer->lookahead == 0) {
+        indent_length = 0;
+        found_end_of_line = true;
+        break;
+      } else {
+        break;
+      }
+    }
+
+    if (found_end_of_line) {
+      if (!indent_length_stack.empty()) {
+        uint16_t current_indent_length = indent_length_stack.back();
+
+        if (
+          valid_symbols[INDENT] &&
+          indent_length > current_indent_length
+        ) {
+          indent_length_stack.push_back(indent_length);
+          lexer->result_symbol = INDENT;
+          return true;
+        }
+
+        if (
+          valid_symbols[DEDENT] &&
+          indent_length < current_indent_length &&
+
+          // Wait to create a dedent token until we've consumed any comments
+          // whose indentation matches the current block.
+          first_comment_indent_length < (int32_t)current_indent_length
+        ) {
+          indent_length_stack.pop_back();
+          lexer->result_symbol = DEDENT;
+          return true;
+        }
+      }
+
+      if (valid_symbols[NEWLINE]) {
+        lexer->result_symbol = NEWLINE;
+        return true;
+      }
+    }
+
+    if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
+      Delimiter delimiter;
+
+      bool has_flags = false;
+      while (lexer->lookahead) {
+        if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
+          delimiter.set_format();
+        } else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
+          delimiter.set_raw();
+        } else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
+          delimiter.set_bytes();
+        } else if (lexer->lookahead != 'u' && lexer->lookahead != 'U') {
+          break;
+        }
+        has_flags = true;
+        advance(lexer);
+      }
+
+      if (lexer->lookahead == '`') {
+        delimiter.set_end_character('`');
+        advance(lexer);
+        lexer->mark_end(lexer);
+      } else if (lexer->lookahead == '\'') {
+        delimiter.set_end_character('\'');
+        advance(lexer);
+        lexer->mark_end(lexer);
+        if (lexer->lookahead == '\'') {
+          advance(lexer);
+          if (lexer->lookahead == '\'') {
+            advance(lexer);
+            lexer->mark_end(lexer);
+            delimiter.set_triple();
+          }
+        }
+      } else if (lexer->lookahead == '"') {
+        delimiter.set_end_character('"');
+        advance(lexer);
+        lexer->mark_end(lexer);
+        if (lexer->lookahead == '"') {
+          advance(lexer);
+          if (lexer->lookahead == '"') {
+            advance(lexer);
+            lexer->mark_end(lexer);
+            delimiter.set_triple();
+          }
+        }
+      }
+
+      if (delimiter.end_character()) {
+        delimiter_stack.push_back(delimiter);
+        lexer->result_symbol = STRING_START;
+        return true;
+      } else if (has_flags) {
+        return false;
+      }
+    }
+
+    return false;
+  }
+
+  vector<uint16_t> indent_length_stack;
+  vector<Delimiter> delimiter_stack;
+};
+
+}
+
+extern "C" {
+
+void *tree_sitter_python_external_scanner_create() {
+  return new Scanner();
+}
+
+bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer,
+                                            const bool *valid_symbols) {
+  Scanner *scanner = static_cast<Scanner *>(payload);
+  return scanner->scan(lexer, valid_symbols);
+}
+
+unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buffer) {
+  Scanner *scanner = static_cast<Scanner *>(payload);
+  return scanner->serialize(buffer);
+}
+
+void tree_sitter_python_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
+  Scanner *scanner = static_cast<Scanner *>(payload);
+  scanner->deserialize(buffer, length);
+}
+
+void tree_sitter_python_external_scanner_destroy(void *payload) {
+  Scanner *scanner = static_cast<Scanner *>(payload);
+  delete scanner;
+}
+
+}

vendor/tree-sitter-python/src/tree_sitter/parser.h

@@ -0,0 +1,223 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+typedef uint16_t TSStateId;
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSFieldMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+struct TSLanguage {
+  uint32_t version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char **symbol_names;
+  const char **field_names;
+  const TSFieldMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+};
+
+/*
+ *  Lexer Macros
+ */
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) id - LARGE_STATE_COUNT
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value            \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = state_value,           \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_val, child_count_val, ...) \
+  {{                                             \
+    .reduce = {                                  \
+      .type = TSParseActionTypeReduce,           \
+      .symbol = symbol_val,                      \
+      .child_count = child_count_val,            \
+      __VA_ARGS__                                \
+    },                                           \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

vendor/tree-sitter-python/test/corpus/expressions.txt

@@ -0,0 +1,753 @@
+=====================================
+Identifiers with Greek letters
+=====================================
+
+ψ1 = β_γ + Ψ_5
+
+---
+
+(module
+  (expression_statement (assignment
+    left: (identifier)
+    right: (binary_operator
+      left: (identifier)
+      right: (identifier)))))
+
+=====================================
+Subscript expressions
+=====================================
+
+a[1]
+b[2, 3]
+c[4, 5,]
+
+---
+
+(module
+  (expression_statement (subscript (identifier) (integer)))
+  (expression_statement (subscript (identifier) (integer) (integer)))
+  (expression_statement (subscript (identifier) (integer) (integer))))
+
+=====================================
+Subscript slice expressions
+=====================================
+
+a[:]
+b[5:]
+b[5:6, ...]
+c[::]
+
+---
+
+(module
+  (expression_statement (subscript
+    (identifier)
+    (slice)))
+  (expression_statement (subscript
+    (identifier)
+    (slice (integer))))
+  (expression_statement (subscript
+    (identifier)
+    (slice (integer) (integer))
+    (ellipsis)))
+  (expression_statement (subscript
+    (identifier)
+    (slice))))
+
+=====================================
+Attribute references
+=====================================
+
+a.b.c
+
+---
+
+(module
+  (expression_statement
+    (attribute
+      (attribute (identifier) (identifier))
+      (identifier))))
+
+=====================================
+Await expressions
+=====================================
+
+await i(j, 5)
+return await i(j, 5)
+
+---
+
+(module
+  (expression_statement
+    (await (call
+      (identifier)
+      (argument_list (identifier) (integer)))))
+  (return_statement
+    (await (call
+      (identifier)
+      (argument_list (identifier) (integer))))))
+
+=====================================
+Call expressions
+=====================================
+
+__a__()
+b(1)
+c(e, f=g)
+i(j, 5,)
+
+---
+
+(module
+  (expression_statement (call
+    (identifier)
+    (argument_list)))
+  (expression_statement (call
+    (identifier)
+    (argument_list (integer))))
+  (expression_statement (call
+    (identifier)
+    (argument_list
+      (identifier)
+      (keyword_argument (identifier) (identifier)))))
+  (expression_statement (call
+    (identifier)
+    (argument_list (identifier) (integer)))))
+
+=====================================
+Print used as an identifier
+=====================================
+
+print()
+print(a)
+print(a, b=c)
+print(d, e)
+print(d, *e)
+print(*f, **g,)
+a(print)
+
+---
+
+(module
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list)))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list (identifier))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (identifier)
+        (keyword_argument (identifier) (identifier)))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (identifier)
+        (list_splat (identifier)))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (list_splat (identifier))
+        (dictionary_splat (identifier)))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list (identifier)))))
+
+=====================================
+Print used as a parameter
+=====================================
+
+def a(print):
+  b
+def a(printer=print):
+  c
+def a(*print):
+  b
+def a(**print):
+  b
+def print():
+  a
+
+---
+
+(module
+  (function_definition
+    (identifier)
+    (parameters (identifier))
+    (block (expression_statement (identifier))))
+  (function_definition
+    (identifier)
+    (parameters (default_parameter (identifier) (identifier)))
+    (block (expression_statement (identifier))))
+  (function_definition
+    (identifier)
+    (parameters (list_splat_pattern (identifier)))
+    (block (expression_statement (identifier))))
+  (function_definition
+    (identifier)
+    (parameters (dictionary_splat_pattern (identifier)))
+    (block (expression_statement (identifier))))
+  (function_definition
+    (identifier)
+    (parameters)
+    (block (expression_statement (identifier)))))
+
+
+=====================================
+Exec used as an identifier
+=====================================
+
+exec("print \"'%s' has %i characters\" % (public_function(), len(public_function()))", {"__builtins__" : None}, safe_dict)
+exec("""exec _code_ in _globs_, _locs_""")
+
+---
+
+(module
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (string
+           (escape_sequence)
+           (escape_sequence))
+        (dictionary (pair (string) (none)))
+        (identifier))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list (string)))))
+
+=====================================
+Async / await used as identifiers
+=====================================
+
+async = 4
+await = 5
+print async, await
+
+---
+
+(module
+  (expression_statement (assignment (identifier) (integer)))
+  (expression_statement (assignment (identifier) (integer)))
+  (print_statement (identifier) (identifier)))
+
+=====================================
+Calls with splats
+=====================================
+
+a(*())
+a(**{})
+a(*b)
+c(d, *e, **g)
+
+---
+
+(module
+  (expression_statement (call
+    (identifier)
+    (argument_list (list_splat (tuple)))))
+  (expression_statement (call
+    (identifier)
+    (argument_list (dictionary_splat (dictionary)))))
+  (expression_statement (call
+    (identifier)
+    (argument_list
+      (list_splat (identifier)))))
+  (expression_statement (call
+    (identifier)
+    (argument_list
+      (identifier)
+      (list_splat (identifier))
+      (dictionary_splat (identifier))))))
+
+=====================================
+Math operators
+=====================================
+
+a + b * c ** d - e / 5
+-5
++x
+~x
+
+---
+
+(module
+  (expression_statement
+    (binary_operator
+      (binary_operator
+        (identifier)
+        (binary_operator
+          (identifier)
+          (binary_operator
+            (identifier)
+            (identifier))))
+      (binary_operator
+        (identifier)
+        (integer))))
+  (expression_statement (unary_operator (integer)))
+  (expression_statement (unary_operator (identifier)))
+  (expression_statement (unary_operator (identifier))))
+
+=====================================
+Binary Addition / Subtraction With Floats
+=====================================
+
+.1-.0
+.1+.0
+.1-0
+.1+0
+
+1-.0
+1+.0
+
+---
+
+(module
+  (expression_statement
+    (binary_operator (float) (float)))
+  (expression_statement
+    (binary_operator (float) (float)))
+  (expression_statement
+    (binary_operator (float) (integer)))
+  (expression_statement
+    (binary_operator (float) (integer)))
+  (expression_statement
+    (binary_operator (integer) (float)))
+  (expression_statement
+    (binary_operator (integer) (float))))
+
+=====================================
+Power Operator Precedence
+=====================================
+
+2**2**3
+-2**2
+
+---
+
+(module
+  (expression_statement
+    (binary_operator
+      (integer)
+      (binary_operator
+        (integer)
+        (integer))))
+  (expression_statement
+    (unary_operator
+      (binary_operator
+        (integer)
+        (integer)))))
+
+=====================================
+Operator precedence
+=====================================
+
+a() + b[c] * c.d.e
+
+---
+
+(module
+  (expression_statement
+    (binary_operator
+      left: (call
+        function: (identifier)
+        arguments: (argument_list))
+      right: (binary_operator
+        left: (subscript
+          value: (identifier)
+          subscript: (identifier))
+        right: (attribute
+          object: (attribute
+            object: (identifier)
+            attribute: (identifier))
+          attribute: (identifier))))))
+
+=====================================
+Bitwise operators
+=====================================
+
+a << b | c >> d & e
+
+---
+
+(module
+  (expression_statement
+    (binary_operator
+      (binary_operator
+        (identifier)
+        (identifier))
+      (binary_operator
+        (binary_operator
+          (identifier)
+          (identifier))
+        (identifier)))))
+
+=====================================
+Boolean operators
+=====================================
+
+a or b and c
+not d
+
+---
+
+(module
+  (expression_statement
+    (boolean_operator
+      (identifier)
+      (boolean_operator
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (not_operator (identifier))))
+
+=====================================
+Comparison operators
+=====================================
+
+a < b <= c == d >= e > f
+not a == b or c == d
+
+---
+
+(module
+  (expression_statement
+    (comparison_operator
+      (identifier)
+      (identifier)
+      (identifier)
+      (identifier)
+      (identifier)
+      (identifier)))
+  (expression_statement
+    (not_operator (boolean_operator
+      (comparison_operator (identifier) (identifier))
+      (comparison_operator (identifier) (identifier))))))
+
+====================================================
+Assignments
+====================================================
+
+a = 1
+a, b = 1, 2
+a, *c = 1, 2, 3
+a, = 1, 2
+a[b] = c = d
+a, *b.c = d
+
+---
+
+(module
+  (expression_statement
+    (assignment
+      (identifier)
+      (integer)))
+  (expression_statement
+    (assignment
+      (pattern_list
+        (identifier)
+        (identifier))
+      (expression_list
+        (integer)
+        (integer))))
+  (expression_statement
+    (assignment
+      (pattern_list
+        (identifier)
+        (list_splat_pattern (identifier)))
+      (expression_list
+        (integer)
+        (integer)
+        (integer))))
+  (expression_statement
+    (assignment
+      (pattern_list
+        (identifier))
+      (expression_list
+        (integer)
+        (integer))))
+  (expression_statement
+    (assignment
+      (subscript (identifier) (identifier))
+      (assignment
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (assignment
+      (pattern_list
+        (identifier)
+        (list_splat_pattern (attribute (identifier) (identifier))))
+      (identifier))))
+
+====================================================
+Assignments with type annotations
+====================================================
+
+tail_leaves: List[Leaf] = []
+
+---
+
+(module
+  (expression_statement (assignment
+    (identifier)
+    (type (subscript (identifier) (identifier)))
+    (list))))
+
+====================================================
+Augmented assignments
+====================================================
+
+a += 1
+b >>= 2
+c //= 1
+
+---
+
+(module
+  (expression_statement
+    (augmented_assignment
+      (identifier)
+      (integer)))
+  (expression_statement
+    (augmented_assignment
+      (identifier)
+      (integer)))
+  (expression_statement
+    (augmented_assignment
+      (identifier)
+      (integer))))
+
+====================================================
+Named expressions
+====================================================
+
+a := x
+(y := f(x))
+foo(x=(y := f(x)))
+y0 = (y1 := f(x))
+def foo(answer=(p := 42)):
+  return answer;
+def foo(answer: (p := 42) = 5):
+  return answer;
+foo(x := 3, cat='vector')
+(z := (y := (x := 0)))
+
+---
+
+(module
+  (expression_statement
+    (named_expression
+      (identifier)
+      (identifier)))
+  (expression_statement
+    (parenthesized_expression
+      (named_expression
+        (identifier)
+        (call (identifier) (argument_list (identifier))))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (keyword_argument
+          (identifier)
+          (parenthesized_expression
+            (named_expression
+              (identifier)
+              (call (identifier) (argument_list (identifier)))))))))
+  (expression_statement
+    (assignment
+      (identifier)
+      (parenthesized_expression
+        (named_expression
+          (identifier)
+          (call (identifier) (argument_list (identifier)))))))
+  (function_definition
+    (identifier)
+    (parameters
+      (default_parameter
+        (identifier)
+        (parenthesized_expression (named_expression (identifier) (integer)))))
+    (block (return_statement (identifier))))
+  (function_definition
+    (identifier)
+    (parameters
+      (typed_default_parameter
+        (identifier)
+        (type (parenthesized_expression (named_expression (identifier) (integer))))
+        (integer)))
+    (block (return_statement (identifier))))
+  (expression_statement
+    (call
+      (identifier)
+      (argument_list
+        (named_expression (identifier) (integer))
+        (keyword_argument (identifier) (string)))))
+  (expression_statement
+    (parenthesized_expression
+      (named_expression
+        (identifier)
+        (parenthesized_expression
+          (named_expression
+            (identifier)
+            (parenthesized_expression
+              (named_expression
+                (identifier)
+                (integer)))))))))
+
+====================================================
+Yield expressions
+====================================================
+
+def example():
+  yield
+  yield 1
+  x = yield 2
+  yield from a
+  yield from (yield from (x for x in range(1, 10)))
+
+---
+
+(module
+  (function_definition (identifier) (parameters) (block
+    (expression_statement (yield))
+    (expression_statement (yield (integer)))
+    (expression_statement
+      (assignment
+        (identifier)
+        (yield (integer))))
+    (expression_statement (yield (identifier)))
+    (expression_statement
+      (yield
+        (parenthesized_expression
+          (yield
+            (generator_expression
+              (identifier)
+              (for_in_clause
+                (identifier)
+                (call (identifier) (argument_list (integer) (integer))))))))))))
+
+====================================================
+lambdas
+====================================================
+
+lambda b, c: d("e" % f)
+lambda: True
+lambda a, b = c, *d, **e: a
+lambda (a, b): (a, b)
+
+---
+
+(module
+  (expression_statement
+    (lambda
+      (lambda_parameters
+        (identifier)
+        (identifier))
+      (call
+        (identifier)
+        (argument_list
+          (binary_operator (string) (identifier))))))
+  (expression_statement
+    (lambda (true)))
+  (expression_statement
+    (lambda
+      (lambda_parameters
+        (identifier)
+        (default_parameter (identifier) (identifier))
+        (list_splat_pattern (identifier))
+        (dictionary_splat_pattern (identifier)))
+      (identifier)))
+  (expression_statement
+    (lambda
+      (lambda_parameters (tuple_pattern (identifier) (identifier)))
+      (tuple (identifier) (identifier)))))
+
+=====================================
+Tuples with splats
+=====================================
+
+(foo, *bar, *baz)
+
+---
+
+(module
+  (expression_statement
+    (tuple (identifier) (list_splat (identifier)) (list_splat (identifier)))))
+
+=====================================
+Tuples with yield
+=====================================
+
+(a, yield a, b, c)
+
+---
+
+(module
+  (expression_statement
+    (tuple
+      (identifier)
+      (yield (expression_list (identifier) (identifier) (identifier))))))
+
+=====================================
+Conditional if expressions
+=====================================
+
+a = b if c else d
+something() if a else d
+slice(1,1,1) if a else d
+
+---
+
+(module
+  (expression_statement
+    (assignment
+      (identifier)
+      (conditional_expression (identifier) (identifier) (identifier))))
+  (expression_statement
+    (conditional_expression (call (identifier) (argument_list)) (identifier) (identifier)))
+  (expression_statement
+    (conditional_expression
+      (call (identifier) (argument_list (integer) (integer) (integer)))
+      (identifier) (identifier))))
+
+========================================
+Async context managers and iterators
+========================================
+
+async with a as b:
+  async for c in d:
+     [e async for f in g]
+
+---
+
+(module
+  (with_statement
+    (with_clause
+      (with_item
+        value: (identifier)
+        alias: (identifier)))
+    body: (block
+      (for_statement
+        left: (identifier)
+        right: (identifier)
+        body: (block
+          (expression_statement
+            (list_comprehension
+              body: (identifier)
+              (for_in_clause
+                left: (identifier)
+                right: (identifier)))))))))

vendor/tree-sitter-python/test/corpus/literals.txt

@@ -0,0 +1,527 @@
+=====================================
+Integers
+=====================================
+
+-1
+0xDEAD
+0XDEAD
+1j
+-1j
+0o123
+0O123
+0b001
+0B001
+1_1
+0B1_1
+0O1_1
+0L
+
+---
+
+(module
+  (expression_statement (unary_operator (integer)))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (unary_operator (integer)))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer))
+  (expression_statement (integer)))
+
+=====================================
+Floats
+=====================================
+
+-.6_6
++.1_1
+123.4123
+123.123J
+1_1.3_1
+1_1.
+1e+3_4j
+.3e1_4
+1_0.l
+.1l
+
+---
+
+(module
+  (expression_statement (unary_operator (float)))
+  (expression_statement (unary_operator (float)))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float)))
+
+
+=====================================
+Scientific Notation Floats
+=====================================
+
+1e322
+1e-3
+1e+3
+1.8e10
+1.e10
+-1e10
+
+---
+
+(module
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (float))
+  (expression_statement (unary_operator (float))))
+
+=====================================
+Strings
+=====================================
+
+"I'm ok"
+'"ok"'
+UR'bye'
+b'sup'
+B"sup"
+`1`
+"\\"
+"/"
+"multiline \
+string"
+b"\x12\u12\U12\x13\N{WINKING FACE}"
+"\xab\123\'\"\a\b\f\r\n\t\v\\"
+"\xgh\o123\p\q\c\d\e\u12\U1234"
+
+---
+
+(module
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string (escape_sequence)))
+  (expression_statement (string))
+  (expression_statement (string (escape_sequence)))
+  (expression_statement (string (escape_sequence) (escape_sequence)))
+  (expression_statement (string (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence) (escape_sequence)))
+  (expression_statement (string)))
+
+=====================================
+Raw strings
+=====================================
+
+'ab\x00cd'
+"\n"
+
+# no escape sequences in these
+r'ab\x00cd'
+ur"\n"
+
+---
+
+(module
+  (expression_statement (string (escape_sequence)))
+  (expression_statement (string (escape_sequence)))
+  (comment)
+  (expression_statement (string))
+  (expression_statement (string)))
+
+=====================================
+Raw strings with escaped quotes
+=====================================
+
+re.compile(r"(\n|\A)#include\s*['\"]"
+           r"(?P<name>[\w\d./\\]+[.]src)['\"]")
+
+---
+
+(module
+  (expression_statement
+    (call
+      (attribute (identifier) (identifier))
+      (argument_list
+        (concatenated_string (string) (string))))))
+
+=====================================
+Format strings
+=====================================
+
+# nested!
+f"a {b(f'c {e} d')} e"
+f"a {{}} e"
+
+---
+
+(module
+  (comment)
+  (expression_statement (string
+    (interpolation
+      (call
+        (identifier)
+        (argument_list
+          (string (interpolation
+            (identifier))))))))
+  (expression_statement (string)))
+
+======================================
+Format strings with format specifiers
+======================================
+
+f"a {b:2} {c:34.5}"
+f"{b:{c.d}.{d.e}}"
+f"{a:#06x}"
+
+---
+
+(module
+  (expression_statement
+    (string
+      (interpolation (identifier) (format_specifier))
+      (interpolation (identifier) (format_specifier))))
+  (expression_statement
+    (string
+      (interpolation
+        (identifier)
+        (format_specifier
+          (format_expression (attribute (identifier) (identifier)))
+          (format_expression (attribute (identifier) (identifier)))))))
+  (expression_statement
+    (string
+      (interpolation (identifier) (format_specifier)))))
+
+=====================================
+Unicode escape sequences
+=====================================
+
+"\x12 \123 \u1234"
+
+---
+
+(module
+  (expression_statement (string
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence))))
+
+=====================================
+Other primitives
+=====================================
+
+True
+False
+None
+
+---
+
+(module
+  (expression_statement (true))
+  (expression_statement (false))
+  (expression_statement (none)))
+
+=====================================
+Concatenated strings
+=====================================
+
+"one" "two" "three"
+
+---
+
+(module
+  (expression_statement
+    (concatenated_string
+      (string)
+      (string)
+      (string))))
+
+=====================================
+Multi-line strings
+=====================================
+
+"""
+A double quote hello,
+without double or single quotes.
+"""
+
+"""
+A double quote "hello",
+with double quotes.
+"""
+
+"""
+A double quote 'hello',
+with single quotes.
+"""
+
+'''
+A single quote hello,
+without double or single quotes.
+'''
+
+'''
+A single quote 'hello',
+with single quotes.
+'''
+
+'''
+A single quote "hello",
+with double quotes.
+'''
+
+"""
+A double quote hello\n\
+with an escaped newline\n\
+and another escaped newline\n\
+"""
+
+---
+
+(module
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string))
+  (expression_statement (string
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence)
+    (escape_sequence))))
+
+=====================================
+Lists
+=====================================
+
+[a, b, [c, d]]
+[*()]
+[*[]]
+[*a]
+[*a.b]
+[*a[b].c]
+[*a()]
+
+---
+
+(module
+  (expression_statement
+    (list
+      (identifier)
+      (identifier)
+      (list
+        (identifier)
+        (identifier))))
+  (expression_statement (list (list_splat (tuple))))
+  (expression_statement (list (list_splat (list))))
+  (expression_statement (list (list_splat (identifier))))
+  (expression_statement (list (list_splat (attribute (identifier) (identifier)))))
+  (expression_statement (list (list_splat (attribute (subscript (identifier) (identifier)) (identifier)))))
+  (expression_statement (list (list_splat (call (identifier) (argument_list))))))
+
+=====================================
+List comprehensions
+=====================================
+
+[a + b for (a, b) in items]
+[a for b in c for a in b]
+[(x,y) for x in [1,2,3] for y in [1,2,3] if True]
+[a for a in lambda: True, lambda: False if a()]
+
+---
+
+(module
+  (expression_statement
+    (list_comprehension
+      (binary_operator (identifier) (identifier))
+      (for_in_clause
+        (tuple_pattern (identifier) (identifier)) (identifier))))
+  (expression_statement
+    (list_comprehension
+      (identifier)
+      (for_in_clause
+        (identifier)
+        (identifier))
+      (for_in_clause
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (list_comprehension
+      (tuple (identifier) (identifier))
+      (for_in_clause (identifier)
+        (list (integer) (integer) (integer)))
+      (for_in_clause (identifier)
+        (list (integer) (integer) (integer)))
+      (if_clause (true))))
+  (expression_statement
+    (list_comprehension
+      (identifier)
+      (for_in_clause (identifier)
+        (lambda (true))
+        (lambda (false)))
+      (if_clause (call (identifier) (argument_list))))))
+
+=====================================
+Dictionaries
+=====================================
+
+{a: 1, b: 2}
+{}
+{**{}}
+{**a}
+{**a.b}
+{**a[b].c}
+{**a()}
+
+---
+
+(module
+  (expression_statement
+    (dictionary
+      (pair (identifier) (integer))
+      (pair (identifier) (integer))))
+  (expression_statement
+    (dictionary))
+  (expression_statement
+    (dictionary (dictionary_splat (dictionary))))
+  (expression_statement
+    (dictionary (dictionary_splat (identifier))))
+  (expression_statement
+    (dictionary (dictionary_splat (attribute (identifier) (identifier)))))
+  (expression_statement
+    (dictionary (dictionary_splat (attribute (subscript (identifier) (identifier)) (identifier)))))
+  (expression_statement
+    (dictionary (dictionary_splat (call (identifier) (argument_list))))))
+
+=====================================
+Dictionary comprehensions
+=====================================
+
+{a: b for a, b in items}
+{a: b for c in d for e in items}
+
+---
+
+(module
+  (expression_statement
+    (dictionary_comprehension
+      (pair (identifier) (identifier))
+      (for_in_clause
+        (pattern_list (identifier) (identifier)) (identifier))))
+  (expression_statement
+    (dictionary_comprehension
+      (pair (identifier) (identifier))
+      (for_in_clause
+        (identifier) (identifier))
+      (for_in_clause
+        (identifier) (identifier)))))
+
+=====================================
+Sets
+=====================================
+
+{a, b, c,}
+{*{}}
+
+---
+
+(module
+  (expression_statement (set (identifier) (identifier) (identifier)))
+  (expression_statement (set (list_splat (dictionary)))))
+
+=====================================
+Set comprehensions
+=====================================
+
+{a[b][c] for a, b, c in items}
+{r for s in qs for n in ms}
+
+---
+
+(module
+  (expression_statement
+    (set_comprehension
+      (subscript (subscript (identifier) (identifier)) (identifier))
+      (for_in_clause
+        (pattern_list (identifier) (identifier) (identifier))
+        (identifier))))
+  (expression_statement
+    (set_comprehension
+      (identifier)
+      (for_in_clause
+        (identifier)
+        (identifier))
+      (for_in_clause
+        (identifier)
+        (identifier)))))
+
+=====================================
+Simple Tuples
+=====================================
+
+()
+(a, b)
+(a, b, c,)
+(print, exec)
+
+---
+
+(module
+  (expression_statement (tuple))
+  (expression_statement (tuple (identifier) (identifier)))
+  (expression_statement (tuple (identifier) (identifier) (identifier)))
+  (expression_statement (tuple (identifier) (identifier))))
+
+=====================================
+Generator expression
+=====================================
+
+(a[b][c] for a, b, c in items)
+dict((a, b) for a, b in d)
+(a for b in c for d in e,)
+(x for x in range(1, 10))
+
+---
+
+(module
+  (expression_statement
+    (generator_expression
+      (subscript (subscript (identifier) (identifier)) (identifier))
+      (for_in_clause
+        (pattern_list (identifier) (identifier) (identifier))
+        (identifier))))
+  (expression_statement
+    (call
+      (identifier)
+      (generator_expression
+        (tuple (identifier) (identifier))
+        (for_in_clause
+          (pattern_list (identifier) (identifier))
+          (identifier)))))
+  (expression_statement
+    (generator_expression
+      (identifier)
+      (for_in_clause
+        (identifier)
+        (identifier))
+      (for_in_clause
+        (identifier)
+        (identifier))))
+  (expression_statement
+    (generator_expression
+      (identifier)
+      (for_in_clause
+        (identifier)
+        (call (identifier) (argument_list (integer) (integer)))))))

vendor/tree-sitter-python/test/corpus/statements.txt

@@ -0,0 +1,958 @@
+=====================================
+Import statements
+=====================================
+
+import a, b
+import b.c as d
+import a.b.c
+
+---
+
+(module
+  (import_statement
+    (dotted_name (identifier))
+    (dotted_name (identifier)))
+  (import_statement
+    (aliased_import
+      (dotted_name (identifier) (identifier))
+      (identifier)))
+  (import_statement
+    (dotted_name (identifier) (identifier) (identifier))))
+
+=====================================
+Import-from statements
+=====================================
+
+from a import b
+from a import *
+from a import (b, c)
+from a.b import c
+from . import b
+from .. import b
+from .a import b
+from ..a import b
+
+---
+
+(module
+  (import_from_statement
+    (dotted_name (identifier))
+    (dotted_name (identifier)))
+  (import_from_statement
+    (dotted_name (identifier))
+    (wildcard_import))
+  (import_from_statement
+    (dotted_name (identifier))
+    (dotted_name (identifier))
+    (dotted_name (identifier)))
+  (import_from_statement
+    (dotted_name (identifier) (identifier))
+    (dotted_name (identifier)))
+  (import_from_statement
+    (relative_import (import_prefix))
+    (dotted_name (identifier)))
+  (import_from_statement
+    (relative_import (import_prefix))
+    (dotted_name (identifier)))
+  (import_from_statement
+    (relative_import
+      (import_prefix)
+      (dotted_name (identifier)))
+      (dotted_name (identifier)))
+  (import_from_statement
+    (relative_import
+      (import_prefix)
+      (dotted_name (identifier)))
+      (dotted_name (identifier))))
+
+=====================================
+Future import statements
+=====================================
+
+from __future__ import print_statement
+from __future__ import python4
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+---
+
+(module
+  (future_import_statement (dotted_name (identifier)))
+  (future_import_statement (dotted_name (identifier)))
+  (future_import_statement
+    (dotted_name (identifier))
+    (dotted_name (identifier))
+    (dotted_name (identifier))
+    (dotted_name (identifier))))
+
+=====================================
+Print statements
+=====================================
+
+print a
+print b, c
+print 0 or 1, 1 or 0,
+print 0 or 1
+
+---
+
+(module
+  (print_statement (identifier))
+  (print_statement (identifier) (identifier))
+  (print_statement
+    (boolean_operator (integer) (integer))
+    (boolean_operator (integer) (integer)))
+  (print_statement
+    (boolean_operator (integer) (integer))))
+
+=====================================
+Print statements with redirection
+=====================================
+
+print >> a
+print >> a, "b", "c"
+
+---
+
+(module
+  (print_statement (chevron (identifier)))
+  (print_statement (chevron (identifier)) (string) (string)))
+
+=====================================
+Assert statements
+=====================================
+
+assert a
+assert b, c
+
+---
+
+(module
+  (assert_statement (identifier))
+  (assert_statement (identifier) (identifier)))
+
+=====================================
+Expression statements
+=====================================
+
+a
+b + c
+1, 2, 3
+1, 2, 3,
+
+---
+
+(module
+  (expression_statement (identifier))
+  (expression_statement (binary_operator (identifier) (identifier)))
+  (expression_statement (integer) (integer) (integer))
+  (expression_statement (integer) (integer) (integer)))
+
+=====================================
+Delete statements
+=====================================
+
+del a[1], b[2]
+
+---
+
+(module
+  (delete_statement (expression_list
+    (subscript (identifier) (integer))
+    (subscript (identifier) (integer)))))
+
+=====================================
+Control-flow statements
+=====================================
+
+while true:
+  pass
+  break
+  continue
+
+---
+
+(module
+  (while_statement
+    condition: (identifier)
+    body: (block
+      (pass_statement)
+      (break_statement)
+      (continue_statement))))
+
+=====================================
+Return statements
+=====================================
+
+return
+return a + b, c
+return not b
+
+---
+
+(module
+  (return_statement)
+  (return_statement (expression_list
+    (binary_operator (identifier) (identifier))
+    (identifier)))
+  (return_statement (not_operator (identifier))))
+
+=====================================
+If statements
+=====================================
+
+if a:
+  b
+  c
+
+---
+
+(module
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (expression_statement (identifier))
+      (expression_statement (identifier)))))
+
+=====================================
+If else statements
+=====================================
+
+if a:
+  b
+elif c:
+  d
+else:
+  f
+
+if a:
+  b
+else:
+  f
+
+if a: b
+
+if a: b; c
+
+---
+
+(module
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (expression_statement (identifier)))
+    alternative: (elif_clause
+      condition: (identifier)
+      consequence: (block
+        (expression_statement (identifier))))
+    alternative: (else_clause
+      body: (block
+        (expression_statement (identifier)))))
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (expression_statement (identifier)))
+    alternative: (else_clause
+      body: (block
+        (expression_statement (identifier)))))
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (expression_statement (identifier))))
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (expression_statement (identifier))
+      (expression_statement (identifier)))))
+
+=====================================
+Nested if statements
+=====================================
+
+if a:
+  if b:
+    c
+  else:
+    if e:
+      f
+g
+
+---
+
+(module
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (if_statement
+        condition: (identifier)
+        consequence: (block
+          (expression_statement (identifier)))
+        alternative: (else_clause
+          body: (block
+            (if_statement
+              condition: (identifier)
+              consequence: (block
+                (expression_statement (identifier)))))))))
+  (expression_statement (identifier)))
+
+=====================================
+While statements
+=====================================
+
+while a:
+  b
+
+while c:
+  d
+else:
+  e
+  f
+
+---
+
+(module
+  (while_statement
+    condition: (identifier)
+    body: (block
+      (expression_statement (identifier))))
+  (while_statement
+    condition: (identifier)
+    body: (block
+      (expression_statement (identifier)))
+    alternative: (else_clause
+      body: (block
+        (expression_statement (identifier))
+        (expression_statement (identifier))))))
+
+=====================================
+For statements
+=====================================
+
+for line, i in lines:
+  print line
+  for character, j in line:
+    print character
+else:
+  print x
+
+for x, in [(1,), (2,), (3,)]:
+  x
+
+---
+
+(module
+  (for_statement
+    left: (pattern_list (identifier) (identifier))
+    right: (identifier)
+    body: (block
+      (print_statement
+        argument: (identifier))
+      (for_statement
+        left: (pattern_list (identifier) (identifier))
+        right: (identifier)
+        body: (block
+          (print_statement
+            argument: (identifier)))))
+    alternative: (else_clause
+      body: (block
+        (print_statement
+          argument: (identifier)))))
+  (for_statement
+    left: (pattern_list (identifier))
+    right: (list (tuple (integer)) (tuple (integer)) (tuple (integer)))
+    body: (block
+      (expression_statement (identifier)))))
+
+=====================================
+Try statements
+=====================================
+
+try:
+  a
+except b:
+  c
+except d as e:
+  f
+except g, h:
+  i
+except:
+  j
+
+try:
+  a
+except b:
+  c
+  d
+else:
+  e
+finally:
+  f
+
+---
+
+(module
+  (try_statement
+    body: (block
+      (expression_statement (identifier)))
+    (except_clause (identifier)
+      (block
+        (expression_statement (identifier))))
+    (except_clause (identifier) (identifier)
+      (block
+        (expression_statement (identifier))))
+    (except_clause (identifier) (identifier)
+      (block
+        (expression_statement (identifier))))
+    (except_clause
+      (block
+        (expression_statement (identifier)))))
+  (try_statement
+    body: (block
+      (expression_statement (identifier)))
+    (except_clause (identifier)
+      (block
+        (expression_statement (identifier))
+        (expression_statement (identifier))))
+    (else_clause body: (block
+      (expression_statement (identifier))))
+    (finally_clause (block
+      (expression_statement (identifier))))))
+
+=====================================
+With statements
+=====================================
+
+with a as b:
+  c
+
+with (open('d') as d,
+      open('e') as e):
+  f
+
+---
+
+(module
+  (with_statement
+    (with_clause
+      (with_item (identifier) (identifier)))
+    (block
+      (expression_statement (identifier))))
+  (with_statement
+    (with_clause
+      (with_item (call (identifier) (argument_list (string))) (identifier))
+      (with_item (call (identifier) (argument_list (string))) (identifier)))
+    (block
+      (expression_statement (identifier)))))
+
+=====================================
+Async Function definitions
+=====================================
+
+async def a():
+  b
+
+async def c(d):
+  e
+
+async def g(g, h,):
+  i
+
+async def c(a: str):
+  a
+
+async def c(a: b.c):
+  a
+
+async def d(a: Sequence[T]) -> T:
+  a
+
+async def i(a, b=c, *c, **d):
+  a
+
+async def d(a: str) -> None:
+  return None
+
+async def d(a:str="default", b=c) -> None:
+  return None
+
+---
+
+(module
+  (function_definition
+    name: (identifier)
+    parameters: (parameters)
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (identifier))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (identifier) (identifier))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (typed_parameter
+        (identifier)
+        type: (type (identifier))))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (typed_parameter
+        (identifier)
+        type: (type (attribute
+          object: (identifier)
+          attribute: (identifier)))))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (typed_parameter
+        (identifier)
+        type: (type (subscript value: (identifier) subscript: (identifier)))))
+    return_type: (type (identifier))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (identifier)
+      (default_parameter
+        name: (identifier)
+        value: (identifier))
+      (list_splat_pattern (identifier))
+      (dictionary_splat_pattern (identifier)))
+    body: (block
+      (expression_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (typed_parameter (identifier) type: (type (identifier))))
+    return_type: (type (none))
+    body: (block
+      (return_statement (none))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (typed_default_parameter
+        name: (identifier)
+        type: (type (identifier))
+        value: (string))
+      (default_parameter
+        name: (identifier)
+        value: (identifier)))
+    return_type: (type (none))
+    body: (block
+      (return_statement (none)))))
+
+=====================================
+Function definitions
+=====================================
+
+def e((a,b)):
+  return (a,b)
+
+def e(*list: str):
+  pass
+
+def e(**list: str):
+  pass
+
+def f():
+  nonlocal a
+
+def g(h, i, *, j, k=100, **kwarg):
+  return h,i,j,k,kwarg
+
+def h(*a):
+  i((*a))
+  j(((*a)))
+
+---
+
+(module
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (tuple_pattern (identifier) (identifier)))
+    body: (block
+      (return_statement (tuple (identifier) (identifier)))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (typed_parameter
+      (list_splat_pattern (identifier))
+      type: (type (identifier))))
+    body: (block
+      (pass_statement)))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (typed_parameter
+      (dictionary_splat_pattern (identifier))
+      type: (type (identifier))))
+    body: (block
+      (pass_statement)))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters)
+    body: (block
+      (nonlocal_statement (identifier))))
+  (function_definition
+    name: (identifier)
+    parameters: (parameters
+      (identifier)
+      (identifier)
+      (list_splat_pattern)
+      (identifier)
+      (default_parameter name: (identifier) value: (integer))
+      (dictionary_splat_pattern (identifier)))
+    body: (block
+      (return_statement (expression_list
+        (identifier)
+        (identifier)
+        (identifier)
+        (identifier)
+        (identifier)))))
+   (function_definition
+     name: (identifier)
+     parameters: (parameters (list_splat_pattern (identifier)))
+     body: (block
+       (expression_statement
+         (call function:
+           (identifier)
+           arguments:
+             (argument_list (parenthesized_expression (list_splat (identifier))))))
+       (expression_statement
+         (call function:
+           (identifier)
+           arguments: (argument_list (parenthesized_expression (parenthesized_expression (list_splat (identifier))))))))))
+
+
+==================================
+Empty blocks
+==================================
+
+# These are not actually valid python; blocks
+# must contain at least one statement. But we
+# allow them because error recovery for empty
+# blocks doesn't work very well otherwise.
+def a(b, c):
+
+if d:
+  print e
+  while f():
+
+---
+
+(module
+  (comment)
+  (comment)
+  (comment)
+  (comment)
+  (function_definition
+    name: (identifier)
+    parameters: (parameters (identifier) (identifier))
+    body: (block))
+  (if_statement
+    condition: (identifier)
+    consequence: (block
+      (print_statement argument: (identifier))
+      (while_statement
+        condition: (call
+          function: (identifier)
+          arguments: (argument_list))
+        body: (block)))))
+
+====================================================
+Class definitions
+====================================================
+
+class A:
+  def b(self):
+    return c
+class B():
+  pass
+class B(method1):
+  def method1(self):
+    return
+class C(method1, Sequence[T]):
+  pass
+class D(Sequence[T, U]):
+  pass
+
+---
+
+(module
+  (class_definition
+    (identifier)
+    (block
+      (function_definition
+        (identifier)
+        (parameters (identifier))
+        (block
+          (return_statement (identifier))))))
+ (class_definition
+    (identifier)
+    (argument_list)
+    (block
+      (pass_statement)))
+ (class_definition
+    (identifier)
+    (argument_list (identifier))
+    (block
+      (function_definition
+        (identifier)
+        (parameters (identifier))
+        (block
+          (return_statement)))))
+ (class_definition
+    (identifier)
+    (argument_list (identifier) (subscript (identifier) (identifier)))
+    (block
+      (pass_statement)))
+ (class_definition
+    (identifier)
+    (argument_list (subscript (identifier) (identifier) (identifier)))
+    (block
+      (pass_statement))))
+
+====================================================
+Class definitions with superclasses
+====================================================
+
+class A(B, C):
+  def d():
+    e
+
+---
+
+(module
+  (class_definition
+    (identifier)
+    (argument_list (identifier) (identifier))
+    (block
+      (function_definition
+        (identifier)
+        (parameters)
+        (block
+          (expression_statement (identifier)))))))
+
+====================================================
+Decorated definitions
+====================================================
+
+@a.b
+class C:
+  @d(1)
+  @e[2].f.g
+  def f():
+    g
+
+  @f()
+  async def f():
+    g
+
+---
+
+(module
+  (decorated_definition
+    (decorator (attribute (identifier) (identifier)))
+    (class_definition (identifier) (block
+      (decorated_definition
+        (decorator (call
+          (identifier)
+          (argument_list (integer))))
+        (decorator (attribute
+          (attribute
+            (subscript
+              (identifier)
+              (integer))
+            (identifier))
+          (identifier)))
+        (function_definition (identifier) (parameters) (block (expression_statement (identifier)))))
+      (decorated_definition
+        (decorator (call (identifier) (argument_list)))
+        (function_definition (identifier) (parameters) (block (expression_statement (identifier)))))))))
+
+
+====================================================
+Raise statements
+====================================================
+
+raise
+raise RuntimeError('NO')
+raise RunTimeError('NO') from e
+
+---
+
+(module
+  (raise_statement)
+  (raise_statement
+    (call (identifier) (argument_list (string))))
+  (raise_statement
+    (call (identifier) (argument_list (string)))
+    (identifier)))
+
+====================================================
+Comments
+====================================================
+
+print a
+# hi
+print b # bye
+print c
+
+---
+
+(module
+  (print_statement (identifier))
+  (comment)
+  (print_statement (identifier))
+  (comment)
+  (print_statement (identifier)))
+
+====================================================
+Comments at different indentation levels
+====================================================
+
+if a:
+  # one
+# two
+    # three
+  b
+    # four
+  c
+
+---
+
+(module
+  (if_statement (identifier)
+    (comment)
+    (comment)
+    (comment)
+    (block
+      (expression_statement (identifier))
+      (comment)
+      (expression_statement (identifier)))))
+
+====================================================
+Comments after dedents
+====================================================
+
+if a:
+  b
+
+# one
+c
+
+---
+
+(module
+  (if_statement
+    (identifier)
+    (block
+      (expression_statement (identifier))))
+  (comment)
+  (expression_statement (identifier)))
+
+====================================================
+Comments at the ends of indented blocks
+====================================================
+
+if a:
+  b
+  # one
+  # two
+
+if c:
+  d
+    # three
+      # four
+
+# five
+
+---
+
+(module
+  (if_statement (identifier) (block
+    (expression_statement (identifier))
+    (comment)
+    (comment)))
+  (if_statement (identifier) (block
+    (expression_statement (identifier))
+    (comment)
+    (comment)))
+  (comment))
+
+====================================================
+Newline tokens followed by comments
+====================================================
+
+print "a"
+  # We need to recognize the newline *preceding* this comment, because there's no newline after it
+---
+
+(module (print_statement (string)) (comment))
+
+====================================================
+Global statements
+====================================================
+
+global a
+global a, b
+
+---
+
+(module
+  (global_statement (identifier))
+  (global_statement (identifier) (identifier)))
+
+====================================================
+Exec statements
+====================================================
+
+exec '1+1'
+exec 'x+=1' in None
+exec 'x+=1' in a, b
+
+---
+
+(module
+  (exec_statement (string))
+  (exec_statement (string) (none))
+  (exec_statement (string) (identifier) (identifier)))
+
+==================================================
+Extra newlines
+==================================================
+
+if a:
+
+
+    b()
+
+
+    c()
+
+
+    def d():
+
+
+        e()
+
+
+    f()
+
+---
+
+(module
+  (if_statement (identifier) (block
+    (expression_statement (call (identifier) (argument_list)))
+    (expression_statement (call (identifier) (argument_list)))
+    (function_definition (identifier) (parameters) (block
+      (expression_statement (call (identifier) (argument_list)))))
+    (expression_statement (call (identifier) (argument_list))))))

vendor/tree-sitter-python/test/highlight/keywords.py

@@ -0,0 +1,30 @@
+if foo():
+# <- keyword
+    pass
+    # <- keyword
+elif bar():
+# <- keyword
+    pass
+else:
+# <- keyword
+    foo
+
+return
+# ^ keyword
+raise e
+# ^ keyword
+
+for i in foo():
+# <- keyword
+#   ^ variable
+#     ^ operator
+#        ^ function
+    continue
+    # <- keyword
+    break
+    # <- keyword
+
+a and b or c
+# ^ operator
+#     ^ variable
+#       ^ operator