sascha
/
Notes
mirror of https://github.com/TriliumNext/Notes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

search refactoring

pull/255/head
zadam 2020-07-21 00:01:07 +07:00
parent f03e4bc7d7
commit 32ecb43b5c
12 changed files with 76 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
spec/search/lexer.spec.js
Unescape Escape View File

@ -1,63 +1,63 @@
const lexer = require('../../src/services/search/lexer.js');
const lex = require('../../src/services/search/services/lex.js');
describe("Lexer fulltext", () => {
it("simple lexing", () => {
expect(lexer("hello world").fulltextTokens.map(t => t.token))
expect(lex("hello world").fulltextTokens.map(t => t.token))
.toEqual(["hello", "world"]);
});
it("use quotes to keep words together", () => {
expect(lexer("'hello world' my friend").fulltextTokens.map(t => t.token))
expect(lex("'hello world' my friend").fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]);
expect(lexer('"hello world" my friend').fulltextTokens.map(t => t.token))
expect(lex('"hello world" my friend').fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]);
expect(lexer('`hello world` my friend').fulltextTokens.map(t => t.token))
expect(lex('`hello world` my friend').fulltextTokens.map(t => t.token))
.toEqual(["hello world", "my", "friend"]);
});
it("you can use different quotes and other special characters inside quotes", () => {
expect(lexer("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token))
expect(lex("'i can use \" or ` or #~=*' without problem").fulltextTokens.map(t => t.token))
.toEqual(["i can use \" or ` or #~=*", "without", "problem"]);
});
it("if quote is not ended then it's just one long token", () => {
expect(lexer("'unfinished quote").fulltextTokens.map(t => t.token))
expect(lex("'unfinished quote").fulltextTokens.map(t => t.token))
.toEqual(["unfinished quote"]);
});
it("parenthesis and symbols in fulltext section are just normal characters", () => {
expect(lexer("what's u=p <b(r*t)h>").fulltextTokens.map(t => t.token))
expect(lex("what's u=p <b(r*t)h>").fulltextTokens.map(t => t.token))
.toEqual(["what's", "u=p", "<b(r*t)h>"]);
});
it("escaping special characters", () => {
expect(lexer("hello \\#\\~\\'").fulltextTokens.map(t => t.token))
expect(lex("hello \\#\\~\\'").fulltextTokens.map(t => t.token))
.toEqual(["hello", "#~'"]);
});
});
describe("Lexer expression", () => {
it("simple attribute existence", () => {
expect(lexer("#label ~relation").expressionTokens.map(t => t.token))
expect(lex("#label ~relation").expressionTokens.map(t => t.token))
.toEqual(["#label", "~relation"]);
});
it("simple label operators", () => {
expect(lexer("#label*=*text").expressionTokens.map(t => t.token))
expect(lex("#label*=*text").expressionTokens.map(t => t.token))
.toEqual(["#label", "*=*", "text"]);
});
it("simple label operator with in quotes and without", () => {
expect(lexer("#label*=*'text'").expressionTokens)
expect(lex("#label*=*'text'").expressionTokens)
.toEqual([
{token: "#label", inQuotes: false},
{token: "*=*", inQuotes: false},
{token: "text", inQuotes: true}
]);
expect(lexer("#label*=*text").expressionTokens)
expect(lex("#label*=*text").expressionTokens)
.toEqual([
{token: "#label", inQuotes: false},
{token: "*=*", inQuotes: false},
@ -66,35 +66,35 @@ describe("Lexer expression", () => {
});
it("complex expressions with and, or and parenthesis", () => {
expect(lexer(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token))
expect(lex(`# (#label=text OR #second=text) AND ~relation`).expressionTokens.map(t => t.token))
.toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "~relation"]);
});
it("dot separated properties", () => {
expect(lexer(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token))
expect(lex(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens.map(t => t.token))
.toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]);
});
it("negation of label and relation", () => {
expect(lexer(`#!capital ~!neighbor`).expressionTokens.map(t => t.token))
expect(lex(`#!capital ~!neighbor`).expressionTokens.map(t => t.token))
.toEqual(["#!capital", "~!neighbor"]);
});
it("negation of sub-expression", () => {
expect(lexer(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token))
expect(lex(`# not(#capital) and note.noteId != "root"`).expressionTokens.map(t => t.token))
.toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]);
});
});
describe("Lexer invalid queries and edge cases", () => {
it("concatenated attributes", () => {
expect(lexer("#label~relation").expressionTokens.map(t => t.token))
expect(lex("#label~relation").expressionTokens.map(t => t.token))
.toEqual(["#label", "~relation"]);
});
it("spaces in attribute names and values", () => {
// invalid but should be reported by parser as an error
expect(lexer(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token))
expect(lex(`#'long label'="hello o' world" ~'long relation'`).expressionTokens.map(t => t.token))
.toEqual(["#long label", "=", "hello o' world", "~long relation"]);
});
});

4
spec/search/parens.spec.js
Unescape Escape View File

@ -1,11 +1,11 @@
const parens = require('../../src/services/search/parens.js');
const handleParens = require('../../src/services/search/services/handle_parens.js');
describe("Parens handler", () => {
it("handles parens", () => {
const input = ["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"]
.map(token => ({token}));
expect(parens(input))
expect(handleParens(input))
.toEqual([
[
{token: "hello"}

32
spec/search/parser.spec.js
Unescape Escape View File

@ -1,5 +1,5 @@
const ParsingContext = require("../../src/services/search/parsing_context.js");
const parser = require('../../src/services/search/parser.js');
const parse = require('../../src/services/search/services/parse.js');
function tokens(...args) {
return args.map(arg => {
@ -17,7 +17,7 @@ function tokens(...args) {
describe("Parser", () => {
it("fulltext parser without content", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: tokens("hello", "hi"),
expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: false})
@ -28,7 +28,7 @@ describe("Parser", () => {
});
it("fulltext parser with content", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: tokens("hello", "hi"),
expressionTokens: [],
parsingContext: new ParsingContext({includeNoteContent: true})
@ -48,7 +48,7 @@ describe("Parser", () => {
});
it("simple label comparison", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#mylabel", "=", "text"),
parsingContext: new ParsingContext()
@ -61,7 +61,7 @@ describe("Parser", () => {
});
it("simple attribute negation", () => {
let rootExp = parser({
let rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#!mylabel"),
parsingContext: new ParsingContext()
@ -72,7 +72,7 @@ describe("Parser", () => {
expect(rootExp.subExpression.attributeType).toEqual("label");
expect(rootExp.subExpression.attributeName).toEqual("mylabel");
rootExp = parser({
rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("~!myrelation"),
parsingContext: new ParsingContext()
@ -85,7 +85,7 @@ describe("Parser", () => {
});
it("simple label AND", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "and", "#second", "=", "text"),
parsingContext: new ParsingContext(true)
@ -102,7 +102,7 @@ describe("Parser", () => {
});
it("simple label AND without explicit AND", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "#second", "=", "text"),
parsingContext: new ParsingContext()
@ -119,7 +119,7 @@ describe("Parser", () => {
});
it("simple label OR", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", "#second", "=", "text"),
parsingContext: new ParsingContext()
@ -136,7 +136,7 @@ describe("Parser", () => {
});
it("fulltext and simple label", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: tokens("hello"),
expressionTokens: tokens("#mylabel", "=", "text"),
parsingContext: new ParsingContext()
@ -153,7 +153,7 @@ describe("Parser", () => {
});
it("label sub-expression", () => {
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "text", "or", tokens("#second", "=", "text", "and", "#third", "=", "text")),
parsingContext: new ParsingContext()
@ -176,11 +176,11 @@ describe("Parser", () => {
});
});
describe("Invalid tokens", () => {
describe("Invalid expressions", () => {
it("incomplete comparison", () => {
const parsingContext = new ParsingContext();
parser({
parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "="),
parsingContext
@ -192,7 +192,7 @@ describe("Invalid tokens", () => {
it("comparison between labels is impossible", () => {
let parsingContext = new ParsingContext();
parser({
parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "#second"),
parsingContext
@ -202,7 +202,7 @@ describe("Invalid tokens", () => {
parsingContext = new ParsingContext();
parser({
parse({
fulltextTokens: [],
expressionTokens: tokens("#first", "=", "note", ".", "relations", "second"),
parsingContext
@ -210,7 +210,7 @@ describe("Invalid tokens", () => {
expect(parsingContext.error).toEqual(`Error near token "note", it's possible to compare with constant only.`);
const rootExp = parser({
const rootExp = parse({
fulltextTokens: [],
expressionTokens: [
{ token: "#first", inQuotes: false },

2
spec/search/search.spec.js
Unescape Escape View File

@ -1,4 +1,4 @@
const searchService = require('../../src/services/search/search.js');
const searchService = require('../../src/services/search/services/search.js');
const Note = require('../../src/services/note_cache/entities/note.js');
const Branch = require('../../src/services/note_cache/entities/branch.js');
const Attribute = require('../../src/services/note_cache/entities/attribute.js');

2
src/routes/api/autocomplete.js
Unescape Escape View File

@ -1,7 +1,7 @@
"use strict";
const noteCacheService = require('../../services/note_cache/note_cache_service');
const searchService = require('../../services/search/search');
const searchService = require('../../services/search/services/search.js');
const repository = require('../../services/repository');
const log = require('../../services/log');
const utils = require('../../services/utils');

2
src/routes/api/search.js
Unescape Escape View File

@ -4,7 +4,7 @@ const repository = require('../../services/repository');
const noteCacheService = require('../../services/note_cache/note_cache.js');
const log = require('../../services/log');
const scriptService = require('../../services/script');
const searchService = require('../../services/search/search');
const searchService = require('../../services/search/services/search.js');
function searchNotes(req) {
const {count, results} = searchService.searchTrimmedNotes(req.params.searchString);

2
src/services/backend_script_api.js
Unescape Escape View File

@ -12,7 +12,7 @@ const dayjs = require('dayjs');
const cloningService = require('./cloning');
const ws = require('./ws.js');
const appInfo = require('./app_info');
const searchService = require('./search/search');
const searchService = require('./search/services/search.js');
/**
* This is the main backend API interface for scripts. It's published in the local "api" object.

0
src/services/search/comparator_builder.js → src/services/search/services/build_comparator.js
Unescape Escape View File

6
src/services/search/parens.js → src/services/search/services/handle_parens.js
Unescape Escape View File

@ -1,7 +1,7 @@
/**
* This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array
*/
function parens(tokens) {
function handle_parens(tokens) {
if (tokens.length === 0) {
return [];
}
@ -34,10 +34,10 @@ function parens(tokens) {
tokens = [
...tokens.slice(0, leftIdx),
parens(tokens.slice(leftIdx + 1, rightIdx)),
handle_parens(tokens.slice(leftIdx + 1, rightIdx)),
...tokens.slice(rightIdx + 1)
];
}
}
module.exports = parens;
module.exports = handle_parens;

4
src/services/search/lexer.js → src/services/search/services/lex.js
Unescape Escape View File

@ -1,4 +1,4 @@
function lexer(str) {
function lex(str) {
str = str.toLowerCase();
const fulltextTokens = [];
@ -130,4 +130,4 @@ function lexer(str) {
}
}
module.exports = lexer;
module.exports = lex;

32
src/services/search/parser.js → src/services/search/services/parse.js
Unescape Escape View File

@ -1,21 +1,21 @@
"use strict";
const AndExp = require('./expressions/and');
const OrExp = require('./expressions/or');
const NotExp = require('./expressions/not');
const ChildOfExp = require('./expressions/child_of');
const DescendantOfExp = require('./expressions/descendant_of');
const ParentOfExp = require('./expressions/parent_of');
const RelationWhereExp = require('./expressions/relation_where');
const PropertyComparisonExp = require('./expressions/property_comparison');
const AttributeExistsExp = require('./expressions/attribute_exists');
const LabelComparisonExp = require('./expressions/label_comparison');
const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext');
const NoteContentProtectedFulltextExp = require('./expressions/note_content_protected_fulltext');
const NoteContentUnprotectedFulltextExp = require('./expressions/note_content_unprotected_fulltext');
const OrderByAndLimitExp = require('./expressions/order_by_and_limit');
const comparatorBuilder = require('./comparator_builder');
const ValueExtractor = require('./value_extractor');
const AndExp = require('../expressions/and.js');
const OrExp = require('../expressions/or.js');
const NotExp = require('../expressions/not.js');
const ChildOfExp = require('../expressions/child_of.js');
const DescendantOfExp = require('../expressions/descendant_of.js');
const ParentOfExp = require('../expressions/parent_of.js');
const RelationWhereExp = require('../expressions/relation_where.js');
const PropertyComparisonExp = require('../expressions/property_comparison.js');
const AttributeExistsExp = require('../expressions/attribute_exists.js');
const LabelComparisonExp = require('../expressions/label_comparison.js');
const NoteCacheFulltextExp = require('../expressions/note_cache_fulltext.js');
const NoteContentProtectedFulltextExp = require('../expressions/note_content_protected_fulltext.js');
const NoteContentUnprotectedFulltextExp = require('../expressions/note_content_unprotected_fulltext.js');
const OrderByAndLimitExp = require('../expressions/order_by_and_limit.js');
const comparatorBuilder = require('./build_comparator.js');
const ValueExtractor = require('../value_extractor.js');
function getFulltext(tokens, parsingContext) {
tokens = tokens.map(t => t.token);

28
src/services/search/search.js → src/services/search/services/search.js
Unescape Escape View File

@ -1,16 +1,16 @@
"use strict";
const lexer = require('./lexer');
const parens = require('./parens');
const parser = require('./parser');
const NoteSet = require("./note_set");
const SearchResult = require("./search_result");
const ParsingContext = require("./parsing_context");
const noteCache = require('../note_cache/note_cache');
const noteCacheService = require('../note_cache/note_cache_service');
const hoistedNoteService = require('../hoisted_note');
const repository = require('../repository');
const utils = require('../utils');
const lex = require('./lex.js');
const handleParens = require('./handle_parens.js');
const parse = require('./parse.js');
const NoteSet = require("../note_set.js");
const SearchResult = require("../search_result.js");
const ParsingContext = require("../parsing_context.js");
const noteCache = require('../../note_cache/note_cache.js');
const noteCacheService = require('../../note_cache/note_cache_service.js');
const hoistedNoteService = require('../../hoisted_note.js');
const repository = require('../../repository.js');
const utils = require('../../utils.js');
/**
* @param {Expression} expression
@ -51,10 +51,10 @@ function findNotesWithExpression(expression) {
}
function parseQueryToExpression(query, parsingContext) {
const {fulltextTokens, expressionTokens} = lexer(query);
const structuredExpressionTokens = parens(expressionTokens);
const {fulltextTokens, expressionTokens} = lex(query);
const structuredExpressionTokens = handleParens(expressionTokens);
const expression = parser({
const expression = parse({
fulltextTokens,
expressionTokens: structuredExpressionTokens,
parsingContext