sascha
/
Notes
mirror of https://github.com/TriliumNext/Notes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

better reporting for search parsing errors

pull/255/head
zadam 2020-07-22 22:52:15 +07:00
parent 60e8bd98b9
commit 3109233d4f
7 changed files with 53 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
package-lock.json generated
Unescape Escape View File

@ -2605,9 +2605,9 @@
} }
}, },
"dayjs": { "dayjs": {
"version": "1.8.29", "version": "1.8.30",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.29.tgz", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.30.tgz",
"integrity": "sha512-Vm6teig8ZWK7rH/lxzVGxZJCljPdmUr6q/3f4fr5F0VWNGVkZEjZOQJsAN8hUHUqn+NK4XHNEpJZS1MwLyDcLw==" "integrity": "sha512-5s5IGuP5bVvIbOWkEDcfmXsUj24fZW1NMHVVSdSFF/kW8d+alZcI9SpBKC+baEyBe+z3fUp17y75ulstv5swUw=="
}, },
"debug": { "debug": {
"version": "4.1.1", "version": "4.1.1",

2
package.json
Unescape Escape View File

@ -31,7 +31,7 @@
"commonmark": "0.29.1", "commonmark": "0.29.1",
"cookie-parser": "1.4.5", "cookie-parser": "1.4.5",
"csurf": "1.11.0", "csurf": "1.11.0",
"dayjs": "1.8.29", "dayjs": "1.8.30",
"debug": "4.1.1", "debug": "4.1.1",
"ejs": "3.1.3", "ejs": "3.1.3",
"electron-debug": "3.1.0", "electron-debug": "3.1.0",

46
spec/search/parser.spec.js
Unescape Escape View File

@ -1,15 +1,19 @@
const ParsingContext = require("../../src/services/search/parsing_context.js"); const ParsingContext = require("../../src/services/search/parsing_context.js");
const parse = require('../../src/services/search/services/parse.js'); const parse = require('../../src/services/search/services/parse.js');
function tokens(...args) { function tokens(toks, cur = 0) {
return args.map(arg => { return toks.map(arg => {
if (Array.isArray(arg)) { if (Array.isArray(arg)) {
return arg; return tokens(arg, cur);
} }
else { else {
cur += arg.length;
return { return {
token: arg, token: arg,
inQuotes: false inQuotes: false,
startIndex: cur - arg.length,
endIndex: cur - 1
}; };
} }
}); });
@ -18,7 +22,7 @@ function tokens(...args) {
describe("Parser", () => { describe("Parser", () => {
it("fulltext parser without content", () => { it("fulltext parser without content", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: tokens("hello", "hi"), fulltextTokens: tokens(["hello", "hi"]),
expressionTokens: [], expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: false}) parsingContext: new ParsingContext({includeNoteContent: false})
}); });
@ -29,7 +33,7 @@ describe("Parser", () => {
it("fulltext parser with content", () => { it("fulltext parser with content", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: tokens("hello", "hi"), fulltextTokens: tokens(["hello", "hi"]),
expressionTokens: [], expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: true}) parsingContext: new ParsingContext({includeNoteContent: true})
}); });
@ -50,7 +54,7 @@ describe("Parser", () => {
it("simple label comparison", () => { it("simple label comparison", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#mylabel", "=", "text"), expressionTokens: tokens(["#mylabel", "=", "text"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -63,7 +67,7 @@ describe("Parser", () => {
it("simple attribute negation", () => { it("simple attribute negation", () => {
let rootExp = parse({ let rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#!mylabel"), expressionTokens: tokens(["#!mylabel"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -74,7 +78,7 @@ describe("Parser", () => {
rootExp = parse({ rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("~!myrelation"), expressionTokens: tokens(["~!myrelation"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -87,7 +91,7 @@ describe("Parser", () => {
it("simple label AND", () => { it("simple label AND", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "and", "#second", "=", "text"), expressionTokens: tokens(["#first", "=", "text", "and", "#second", "=", "text"]),
parsingContext: new ParsingContext(true) parsingContext: new ParsingContext(true)
}); });
@ -104,7 +108,7 @@ describe("Parser", () => {
it("simple label AND without explicit AND", () => { it("simple label AND without explicit AND", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "#second", "=", "text"), expressionTokens: tokens(["#first", "=", "text", "#second", "=", "text"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -121,7 +125,7 @@ describe("Parser", () => {
it("simple label OR", () => { it("simple label OR", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", "#second", "=", "text"), expressionTokens: tokens(["#first", "=", "text", "or", "#second", "=", "text"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -137,8 +141,8 @@ describe("Parser", () => {
it("fulltext and simple label", () => { it("fulltext and simple label", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: tokens("hello"), fulltextTokens: tokens(["hello"]),
expressionTokens: tokens("#mylabel", "=", "text"), expressionTokens: tokens(["#mylabel", "=", "text"]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -155,7 +159,7 @@ describe("Parser", () => {
it("label sub-expression", () => { it("label sub-expression", () => {
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", tokens("#second", "=", "text", "and", "#third", "=", "text")), expressionTokens: tokens(["#first", "=", "text", "or", ["#second", "=", "text", "and", "#third", "=", "text"]]),
parsingContext: new ParsingContext() parsingContext: new ParsingContext()
}); });
@ -182,7 +186,7 @@ describe("Invalid expressions", () => {
parse({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "="), expressionTokens: tokens(["#first", "="]),
parsingContext parsingContext
}); });
@ -191,24 +195,26 @@ describe("Invalid expressions", () => {
it("comparison between labels is impossible", () => { it("comparison between labels is impossible", () => {
let parsingContext = new ParsingContext(); let parsingContext = new ParsingContext();
parsingContext.originalQuery = "#first = #second";
parse({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "#second"), expressionTokens: tokens(["#first", "=", "#second"]),
parsingContext parsingContext
}); });
expect(parsingContext.error).toEqual(`Error near token "#second", it's possible to compare with constant only.`); expect(parsingContext.error).toEqual(`Error near token "#second" in "#first = #second", it's possible to compare with constant only.`);
parsingContext = new ParsingContext(); parsingContext = new ParsingContext();
parsingContext.originalQuery = "#first = note.relations.second";
parse({ parse({
fulltextTokens: [], fulltextTokens: [],
expressionTokens: tokens("#first", "=", "note", ".", "relations", "second"), expressionTokens: tokens(["#first", "=", "note", ".", "relations", "second"]),
parsingContext parsingContext
}); });
expect(parsingContext.error).toEqual(`Error near token "note", it's possible to compare with constant only.`); expect(parsingContext.error).toEqual(`Error near token "note" in "#first = note.relations.s...", it's possible to compare with constant only.`);
const rootExp = parse({ const rootExp = parse({
fulltextTokens: [], fulltextTokens: [],

1
src/services/search/parsing_context.js
Unescape Escape View File

@ -5,6 +5,7 @@ class ParsingContext {
this.includeNoteContent = !!params.includeNoteContent; this.includeNoteContent = !!params.includeNoteContent;
this.fuzzyAttributeSearch = !!params.fuzzyAttributeSearch; this.fuzzyAttributeSearch = !!params.fuzzyAttributeSearch;
this.highlightedTokens = []; this.highlightedTokens = [];
this.originalQuery = "";
this.error = null; this.error = null;
} }

6
src/services/search/services/handle_parens.js
Unescape Escape View File

@ -1,7 +1,7 @@
/** /**
* This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array * This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array
*/ */
function handle_parens(tokens) { function handleParens(tokens) {
if (tokens.length === 0) { if (tokens.length === 0) {
return []; return [];
} }
@ -34,10 +34,10 @@ function handle_parens(tokens) {
tokens = [ tokens = [
...tokens.slice(0, leftIdx), ...tokens.slice(0, leftIdx),
handle_parens(tokens.slice(leftIdx + 1, rightIdx)), handleParens(tokens.slice(leftIdx + 1, rightIdx)),
...tokens.slice(rightIdx + 1) ...tokens.slice(rightIdx + 1)
]; ];
} }
} }
module.exports = handle_parens; module.exports = handleParens;

24
src/services/search/services/parse.js
Unescape Escape View File

@ -51,6 +51,16 @@ function getExpression(tokens, parsingContext, level = 0) {
let i; let i;
function context(i) {
let {startIndex, endIndex} = tokens[i];
startIndex = Math.max(0, startIndex - 20);
endIndex = Math.min(parsingContext.originalQuery.length, endIndex + 20);
return '"' + (startIndex !== 0 ? "..." : "")
+ parsingContext.originalQuery.substr(startIndex, endIndex - startIndex)
+ (endIndex !== parsingContext.originalQuery.length ? "..." : "") + '"';
}
function parseNoteProperty() { function parseNoteProperty() {
if (tokens[i].token !== '.') { if (tokens[i].token !== '.') {
parsingContext.addError('Expected "." to separate field path'); parsingContext.addError('Expected "." to separate field path');
@ -65,7 +75,7 @@ function getExpression(tokens, parsingContext, level = 0) {
const operator = tokens[i].token; const operator = tokens[i].token;
if (!isOperator(operator)) { if (!isOperator(operator)) {
parsingContext.addError(`After content expected operator, but got "${tokens[i].token}"`); parsingContext.addError(`After content expected operator, but got "${tokens[i].token}" in ${context(i)}`);
return; return;
} }
@ -97,7 +107,7 @@ function getExpression(tokens, parsingContext, level = 0) {
if (tokens[i].token === 'labels') { if (tokens[i].token === 'labels') {
if (tokens[i + 1].token !== '.') { if (tokens[i + 1].token !== '.') {
parsingContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}"`); parsingContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}" in ${context(i)}`);
return; return;
} }
@ -108,7 +118,7 @@ function getExpression(tokens, parsingContext, level = 0) {
if (tokens[i].token === 'relations') { if (tokens[i].token === 'relations') {
if (tokens[i + 1].token !== '.') { if (tokens[i + 1].token !== '.') {
parsingContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}"`); parsingContext.addError(`Expected "." to separate field path, got "${tokens[i + 1].token}" in ${context(i)}`);
return; return;
} }
@ -124,7 +134,7 @@ function getExpression(tokens, parsingContext, level = 0) {
const comparator = comparatorBuilder(operator, comparedValue); const comparator = comparatorBuilder(operator, comparedValue);
if (!comparator) { if (!comparator) {
parsingContext.addError(`Can't find operator '${operator}'`); parsingContext.addError(`Can't find operator '${operator}' in ${context(i)}`);
return; return;
} }
@ -133,7 +143,7 @@ function getExpression(tokens, parsingContext, level = 0) {
return new PropertyComparisonExp(propertyName, comparator); return new PropertyComparisonExp(propertyName, comparator);
} }
parsingContext.addError(`Unrecognized note property "${tokens[i].token}"`); parsingContext.addError(`Unrecognized note property "${tokens[i].token}" in ${context(i)}`);
} }
function parseAttribute(name) { function parseAttribute(name) {
@ -161,7 +171,7 @@ function getExpression(tokens, parsingContext, level = 0) {
if (!tokens[i + 2].inQuotes if (!tokens[i + 2].inQuotes
&& (comparedValue.startsWith('#') || comparedValue.startsWith('~') || comparedValue === 'note')) { && (comparedValue.startsWith('#') || comparedValue.startsWith('~') || comparedValue === 'note')) {
parsingContext.addError(`Error near token "${comparedValue}", it's possible to compare with constant only.`); parsingContext.addError(`Error near token "${comparedValue}" in ${context(i)}, it's possible to compare with constant only.`);
return; return;
} }
@ -174,7 +184,7 @@ function getExpression(tokens, parsingContext, level = 0) {
const comparator = comparatorBuilder(operator, comparedValue); const comparator = comparatorBuilder(operator, comparedValue);
if (!comparator) { if (!comparator) {
parsingContext.addError(`Can't find operator '${operator}'`); parsingContext.addError(`Can't find operator '${operator}' in ${context(i)}`);
} else { } else {
i += 2; i += 2;

3
src/services/search/services/search.js
Unescape Escape View File

@ -57,7 +57,8 @@ function parseQueryToExpression(query, parsingContext) {
const expression = parse({ const expression = parse({
fulltextTokens, fulltextTokens,
expressionTokens: structuredExpressionTokens, expressionTokens: structuredExpressionTokens,
parsingContext parsingContext,
originalQuery: query
}); });
return expression; return expression;