mirror of https://github.com/TriliumNext/Notes
preparing 0.59 without ocr/pdf, userguide, note ancillaries
zadam
parent
42e08284b0
commit
6f7b554cdc
@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
rm -rf ./tmp/api_docs/backend_api
|
||||
rm -rf ./tmp/api_docs/frontend_api
|
||||
|
||||
./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./tmp/api_docs/backend_api src/becca/entities/*.js \
|
||||
src/services/backend_script_api.js src/services/sql.js
|
||||
|
||||
./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./tmp/api_docs/frontend_api src/public/app/entities/*.js \
|
||||
src/public/app/services/frontend_script_api.js src/public/app/widgets/right_panel_widget.js
|
||||
|
||||
rm -rf ./docs/api_docs/backend_api ./docs/api_docs/frontend_api
|
||||
|
||||
node src/transform_api_docs.js
|
||||
|
||||
rm -rf ./docs/api_docs/fonts ./docs/api_docs/styles ./docs/api_docs/scripts ./docs/api_docs/backend_api/index.html ./docs/api_docs/frontend_api/index.html
|
||||
@ -1,20 +0,0 @@
|
||||
CREATE TABLE IF NOT EXISTS "note_ancillaries"
|
||||
(
|
||||
noteAncillaryId TEXT not null primary key,
|
||||
noteId TEXT not null,
|
||||
name TEXT not null,
|
||||
mime TEXT not null,
|
||||
isProtected INT not null DEFAULT 0,
|
||||
contentCheckSum TEXT not null,
|
||||
utcDateModified TEXT not null,
|
||||
isDeleted INT not null,
|
||||
`deleteId` TEXT DEFAULT NULL);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "note_ancillary_contents" (`noteAncillaryId` TEXT NOT NULL PRIMARY KEY,
|
||||
`content` TEXT DEFAULT NULL,
|
||||
`utcDateModified` TEXT NOT NULL);
|
||||
|
||||
CREATE INDEX IDX_note_ancillaries_name
|
||||
on note_ancillaries (name);
|
||||
CREATE UNIQUE INDEX IDX_note_ancillaries_noteId_name
|
||||
on note_ancillaries (noteId, name);
|
||||
@ -1,39 +0,0 @@
|
||||
module.exports = async () => {
|
||||
const cls = require("../../src/services/cls");
|
||||
const beccaLoader = require("../../src/becca/becca_loader");
|
||||
const becca = require("../../src/becca/becca");
|
||||
const log = require("../../src/services/log");
|
||||
|
||||
await cls.init(async () => {
|
||||
beccaLoader.load();
|
||||
|
||||
for (const note of Object.values(becca.notes)) {
|
||||
if (note.type !== 'canvas') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (note.isProtected) {
|
||||
// can't migrate protected notes, but that's not critical.
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = note.getContent(true);
|
||||
let svg;
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(content);
|
||||
svg = payload?.svg;
|
||||
|
||||
if (!svg) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
log.info(`Could not create a note ancillary for canvas "${note.noteId}" with error: ${e.message} ${e.stack}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
note.saveNoteAncillary('canvasSvg', 'image/svg+xml', svg);
|
||||
}
|
||||
});
|
||||
};
|
||||
@ -1,8 +1,7 @@
|
||||
{
|
||||
"templates": {
|
||||
"default": {
|
||||
"includeDate": false,
|
||||
"outputSourceFiles": false
|
||||
"includeDate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,129 +0,0 @@
|
||||
const fs = require("fs-extra");
|
||||
const utils = require("../../src/services/utils");
|
||||
const html = require("html");
|
||||
|
||||
const SRC_DIR = './src-build/docs-website';
|
||||
const USER_GUIDE_DIR = './docs/user_guide';
|
||||
const META_PATH = USER_GUIDE_DIR + '/!!!meta.json';
|
||||
const WEB_TMP_DIR = './tmp/user_guide_web';
|
||||
fs.copySync(USER_GUIDE_DIR, WEB_TMP_DIR);
|
||||
|
||||
const meta = JSON.parse(readFile(META_PATH));
|
||||
const rootNoteMeta = meta.files[0];
|
||||
const noteIdToMeta = {};
|
||||
createNoteIdToMetaMapping(rootNoteMeta);
|
||||
|
||||
addNavigationAndStyle(rootNoteMeta, WEB_TMP_DIR);
|
||||
|
||||
fs.writeFileSync(WEB_TMP_DIR + '/main.js', readFile(SRC_DIR + "/main.js"));
|
||||
fs.writeFileSync(WEB_TMP_DIR + '/main.css', readFile(SRC_DIR + "/main.css"));
|
||||
fs.cpSync('libraries/ckeditor/ckeditor-content.css' ,WEB_TMP_DIR + '/ckeditor-content.css');
|
||||
|
||||
function addNavigationAndStyle(noteMeta, parentDirPath) {
|
||||
const nav = createNavigation(rootNoteMeta, noteMeta);
|
||||
|
||||
if (noteMeta.dataFileName) {
|
||||
const filePath = parentDirPath + "/" + noteMeta.dataFileName;
|
||||
|
||||
console.log(`Adding nav to ${filePath}`);
|
||||
|
||||
const content = readFile(filePath);
|
||||
const depth = noteMeta.notePath.length - 1;
|
||||
const updatedContent = content
|
||||
.replaceAll("</head>", `
|
||||
<link rel="stylesheet" href="${"../".repeat(depth)}main.css">
|
||||
<link rel="stylesheet" href="${"../".repeat(depth)}ckeditor-content.css">
|
||||
<script src="${"../".repeat(depth)}main.js"></script>`)
|
||||
.replaceAll("</body>", nav + "</body>");
|
||||
const prettified = html.prettyPrint(updatedContent, {indent_size: 2});
|
||||
fs.writeFileSync(filePath, prettified);
|
||||
}
|
||||
|
||||
for (const childNoteMeta of noteMeta.children || []) {
|
||||
addNavigationAndStyle(childNoteMeta, parentDirPath + '/' + noteMeta.dirFileName);
|
||||
}
|
||||
}
|
||||
|
||||
function createNavigation(rootMeta, sourceMeta) {
|
||||
function saveNavigationInner(meta, parentNoteId = 'root') {
|
||||
let html = `<li data-branch-id="${parentNoteId}_${meta.noteId}">`;
|
||||
|
||||
const escapedTitle = utils.escapeHtml(`${meta.prefix ? `${meta.prefix} - ` : ''}${meta.title}`);
|
||||
|
||||
if (meta.dataFileName) {
|
||||
const targetUrl = getTargetUrl(meta.noteId, sourceMeta);
|
||||
|
||||
html += `<a href="${targetUrl}">${escapedTitle}</a>`;
|
||||
}
|
||||
else {
|
||||
html += escapedTitle;
|
||||
}
|
||||
|
||||
if (meta.children && meta.children.length > 0) {
|
||||
html += '<ul>';
|
||||
|
||||
for (const child of meta.children) {
|
||||
html += saveNavigationInner(child, meta.noteId);
|
||||
}
|
||||
|
||||
html += '</ul>'
|
||||
}
|
||||
|
||||
return `${html}</li>`;
|
||||
}
|
||||
|
||||
return `<nav class="note-tree-nav"><ul>${saveNavigationInner(rootMeta)}</ul></nav>`;
|
||||
}
|
||||
|
||||
function createNoteIdToMetaMapping(noteMeta) {
|
||||
noteIdToMeta[noteMeta.noteId] = noteMeta;
|
||||
|
||||
for (const childNoteMeta of noteMeta.children || []) {
|
||||
createNoteIdToMetaMapping(childNoteMeta);
|
||||
}
|
||||
}
|
||||
|
||||
function getTargetUrl(targetNoteId, sourceMeta) {
|
||||
const targetMeta = noteIdToMeta[targetNoteId];
|
||||
|
||||
if (!targetMeta) {
|
||||
throw new Error(`Could not find note meta for noteId '${targetNoteId}'`);
|
||||
}
|
||||
|
||||
const targetPath = targetMeta.notePath.slice();
|
||||
const sourcePath = sourceMeta.notePath.slice();
|
||||
|
||||
// > 1 for edge case that targetPath and sourcePath are exact same (link to itself)
|
||||
while (targetPath.length > 1 && sourcePath.length > 1 && targetPath[0] === sourcePath[0]) {
|
||||
targetPath.shift();
|
||||
sourcePath.shift();
|
||||
}
|
||||
|
||||
let url = "../".repeat(sourcePath.length - 1);
|
||||
|
||||
for (let i = 0; i < targetPath.length - 1; i++) {
|
||||
const meta = noteIdToMeta[targetPath[i]];
|
||||
|
||||
if (!meta) {
|
||||
throw new Error(`Cannot resolve note '${targetPath[i]}' from path '${targetPath.toString()}'`);
|
||||
}
|
||||
|
||||
url += `${encodeURIComponent(meta.dirFileName)}/`;
|
||||
}
|
||||
|
||||
const targetPathNoteId = targetPath[targetPath.length - 1];
|
||||
const meta = noteIdToMeta[targetPathNoteId];
|
||||
|
||||
if (!meta) {
|
||||
throw new Error(`Cannot resolve note '${targetPathNoteId}' from path '${targetPath.toString()}'`);
|
||||
}
|
||||
|
||||
// link can target note which is only "folder-note" and as such will not have a file in an export
|
||||
url += encodeURIComponent(meta.dataFileName || meta.dirFileName);
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
function readFile(filePath) {
|
||||
return fs.readFileSync(filePath).toString();
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
body {
|
||||
display: flex;
|
||||
flex-direction: row-reverse;
|
||||
width: 1100px;
|
||||
margin: auto;
|
||||
font-family: 'Lucida Grande', 'Lucida Sans Unicode', arial, sans-serif;
|
||||
}
|
||||
|
||||
.note-tree-nav {
|
||||
padding-top: 10px;
|
||||
width: 300px;
|
||||
margin-right: 20px;
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.note-tree-nav ul {
|
||||
padding-left: 20px;
|
||||
list-style-type: none;
|
||||
}
|
||||
|
||||
.note-tree-nav ul li {
|
||||
line-height: 150%;
|
||||
font-size: 105%;
|
||||
}
|
||||
|
||||
.note-tree-nav > ul > li > a {
|
||||
font-size: x-large;
|
||||
}
|
||||
|
||||
.note-tree-nav a {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.note-tree-nav li span.expander, .note-tree-nav li span.spacer {
|
||||
width: 1em;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.note-tree-nav li span.expander {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.content {
|
||||
width: 780px;
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
for (const li of document.querySelectorAll('.note-tree-nav li')) {
|
||||
const branchId = li.getAttribute("data-branch-id");
|
||||
if (branchId.startsWith("root_")) {
|
||||
// first level is expanded and cannot be collapsed
|
||||
continue;
|
||||
}
|
||||
|
||||
const newDiv = document.createElement("span");
|
||||
const subList = li.querySelector('ul');
|
||||
|
||||
if (subList) {
|
||||
const toggleVisibility = (show) => {
|
||||
newDiv.innerHTML = show ? "▾ " : "▸ ";
|
||||
subList.style.display = show ? 'block' : 'none';
|
||||
|
||||
localStorage.setItem(branchId, show ? "true" : "false");
|
||||
};
|
||||
|
||||
newDiv.classList.add("expander");
|
||||
newDiv.addEventListener('click', () => toggleVisibility(subList.style.display === 'none'));
|
||||
|
||||
toggleVisibility(localStorage.getItem(branchId) === "true");
|
||||
} else {
|
||||
newDiv.classList.add("spacer");
|
||||
}
|
||||
|
||||
li.prepend(newDiv);
|
||||
}
|
||||
}, false);
|
||||
@ -1,12 +0,0 @@
|
||||
const fs = require("fs");
|
||||
|
||||
const PACKAGE_JSON_PATH = './node_modules/pdfjs-dist/package.json';
|
||||
|
||||
const packageJson = JSON.parse(
|
||||
fs.readFileSync(PACKAGE_JSON_PATH).toString()
|
||||
);
|
||||
|
||||
// non-legacy build doesn't work on node 16 at least
|
||||
packageJson.main = "legacy/build/pdf.js";
|
||||
|
||||
fs.writeFileSync(PACKAGE_JSON_PATH, JSON.stringify(packageJson, null, 2));
|
||||
@ -1,260 +0,0 @@
|
||||
const sanitizeHtml = require('sanitize-html');
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const html = require("html");
|
||||
|
||||
const TMP_API_DOCS = './tmp/api_docs';
|
||||
const TMP_FE_DOCS = TMP_API_DOCS + '/frontend_api';
|
||||
const TMP_BE_DOCS = TMP_API_DOCS + '/backend_api';
|
||||
|
||||
const sourceFiles = getFilesRecursively(TMP_API_DOCS);
|
||||
|
||||
for (const sourcePath of sourceFiles) {
|
||||
const content = fs.readFileSync(sourcePath).toString();
|
||||
const transformedContent = transform(content);
|
||||
const prettifiedContent = html.prettyPrint(transformedContent, {indent_size: 2});
|
||||
const filteredContent = prettifiedContent
|
||||
.replace(/<br \/>Documentation generated by <a href="https:\/\/github.com\/jsdoc\/jsdoc">[^<]+<\/a>/gi, '')
|
||||
.replace(/JSDoc: (Class|Module): [a-z]+/gi, '');
|
||||
|
||||
const destPath = sourcePath.replaceAll("tmp", "docs");
|
||||
|
||||
fs.mkdirSync(path.dirname(destPath), {recursive: true});
|
||||
fs.writeFileSync(destPath, filteredContent.trim());
|
||||
|
||||
console.log(destPath);
|
||||
}
|
||||
|
||||
const USER_GUIDE_DIR = './docs/user_guide';
|
||||
const META_PATH = USER_GUIDE_DIR + '/!!!meta.json';
|
||||
|
||||
const meta = JSON.parse(fs.readFileSync(META_PATH).toString());
|
||||
const rootNoteMeta = meta.files[0];
|
||||
|
||||
const {noteMeta: scriptApiDocsRoot, filePath: scriptApiDocsRootFilePath, notePath: scriptApiDocsRootNotePath} =
|
||||
findNoteMeta(rootNoteMeta, 'Script API', []);
|
||||
const BE_FILES = ['AbstractBeccaEntity', 'BAttribute', 'BBranch', 'BEtapiToken', 'BNote', 'BNoteRevision', 'BOption', 'BRecentNote', 'module-sql'];
|
||||
|
||||
const FE_FILES = ['FNote', 'FAttribute', 'FBranch', 'FNoteComplement'];
|
||||
|
||||
scriptApiDocsRoot.dirFileName = scriptApiDocsRoot.dataFileName.substr(0, scriptApiDocsRoot.dataFileName.length - 5);
|
||||
scriptApiDocsRoot.children = getScriptApiMeta();
|
||||
|
||||
fs.writeFileSync(META_PATH, JSON.stringify(meta, null, 2));
|
||||
const scriptApiDocsRootDir = USER_GUIDE_DIR + scriptApiDocsRootFilePath;
|
||||
|
||||
fs.mkdirSync(scriptApiDocsRootDir, {recursive: true});
|
||||
fs.mkdirSync(scriptApiDocsRootDir + '/BackendScriptApi', {recursive: true});
|
||||
fs.mkdirSync(scriptApiDocsRootDir + '/FrontendScriptApi', {recursive: true});
|
||||
|
||||
const BE_ROOT = scriptApiDocsRootDir + '/BackendScriptApi.html';
|
||||
const FE_ROOT = scriptApiDocsRootDir + '/FrontendScriptApi.html';
|
||||
|
||||
fs.copyFileSync(TMP_BE_DOCS + '/BackendScriptApi.html', BE_ROOT);
|
||||
fs.copyFileSync(TMP_FE_DOCS + '/FrontendScriptApi.html', FE_ROOT);
|
||||
|
||||
for (const file of BE_FILES) {
|
||||
fs.copyFileSync(TMP_BE_DOCS + '/' + file + '.html', `${scriptApiDocsRootDir}/BackendScriptApi/${file}.html`);
|
||||
}
|
||||
rewriteLinks(BE_ROOT, BE_FILES, 'BackendScriptApi');
|
||||
|
||||
for (const file of FE_FILES) {
|
||||
fs.copyFileSync(TMP_FE_DOCS + '/' + file + '.html', `${scriptApiDocsRootDir}/FrontendScriptApi/${file}.html`);
|
||||
}
|
||||
rewriteLinks(FE_ROOT, FE_FILES, 'FrontendScriptApi');
|
||||
|
||||
fs.rmSync(USER_GUIDE_DIR + '/index.html', {force: true});
|
||||
fs.rmSync(USER_GUIDE_DIR + '/navigation.html', {force: true});
|
||||
fs.rmSync(USER_GUIDE_DIR + '/style.css', {force: true});
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
function getFilesRecursively(directory) {
|
||||
const files = [];
|
||||
|
||||
function getFilesRecursivelyInner(directory) {
|
||||
const filesInDirectory = fs.readdirSync(directory);
|
||||
for (const file of filesInDirectory) {
|
||||
const absolute = path.join(directory, file);
|
||||
if (fs.statSync(absolute).isDirectory()) {
|
||||
getFilesRecursivelyInner(absolute);
|
||||
} else if (file.endsWith('.html')) {
|
||||
files.push(absolute);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getFilesRecursivelyInner(directory);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
function transform(content) {
|
||||
const result = sanitizeHtml(content, {
|
||||
allowedTags: [
|
||||
'h4', 'h5', 'h6', 'blockquote', 'p', 'a', 'ul', 'ol',
|
||||
'li', 'b', 'i', 'strong', 'em', 'strike', 's', 'del', 'abbr', 'code', 'hr', 'br', 'div',
|
||||
'table', 'thead', 'caption', 'tbody', 'tr', 'th', 'td', 'pre', 'section', 'img',
|
||||
'figure', 'figcaption', 'span', 'label', 'input',
|
||||
],
|
||||
nonTextTags: [ 'style', 'script', 'textarea', 'option', 'h1', 'h2', 'h3', 'nav' ],
|
||||
allowedAttributes: {
|
||||
'a': [ 'href', 'class', 'data-note-path' ],
|
||||
'img': [ 'src' ],
|
||||
'section': [ 'class', 'data-note-id' ],
|
||||
'figure': [ 'class' ],
|
||||
'span': [ 'class', 'style' ],
|
||||
'label': [ 'class' ],
|
||||
'input': [ 'class', 'type', 'disabled' ],
|
||||
'code': [ 'class' ],
|
||||
'ul': [ 'class' ],
|
||||
'table': [ 'class' ],
|
||||
'en-media': [ 'hash' ]
|
||||
},
|
||||
allowedSchemes: ['http', 'https', 'ftp', 'mailto', 'data', 'evernote'],
|
||||
transformTags: {
|
||||
// 'h5': sanitizeHtml.simpleTransform('strong', {}, false),
|
||||
'table': sanitizeHtml.simpleTransform('table', {}, false)
|
||||
},
|
||||
});
|
||||
|
||||
return result.replace(/<table>/gi, '<figure class="table"><table>')
|
||||
.replace(/<\/table>/gi, '</table></figure>')
|
||||
.replace(/<div><\/div>/gi, '')
|
||||
.replace(/<h5>/gi, '<p><strong>')
|
||||
.replace(/<\/h5>/gi, '</strong></p>')
|
||||
.replace(/<h4>/gi, '<h2>')
|
||||
.replace(/<\/h4>/gi, '</h2>')
|
||||
.replace(/<span class="signature-attributes">opt<\/span>/gi, '')
|
||||
.replace(/<h2>.*new (BackendScriptApi|FrontendScriptApi).*<\/h2>/gi, '')
|
||||
;
|
||||
}
|
||||
|
||||
function findNoteMeta(noteMeta, name, notePath) {
|
||||
if (noteMeta.title === name) {
|
||||
return {
|
||||
noteMeta,
|
||||
filePath: '/' + noteMeta.dirFileName,
|
||||
notePath
|
||||
};
|
||||
}
|
||||
|
||||
for (const childMeta of noteMeta.children || []) {
|
||||
const ret = findNoteMeta(childMeta, name, [...notePath, childMeta.noteId]);
|
||||
|
||||
if (ret) {
|
||||
return {
|
||||
noteMeta: ret.noteMeta,
|
||||
filePath: '/' + noteMeta.dirFileName + ret.filePath,
|
||||
notePath: ret.notePath
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function rewriteLinks(rootFilePath, files, dir) {
|
||||
let content = fs.readFileSync(rootFilePath).toString();
|
||||
|
||||
for (const file of files) {
|
||||
content = content.replaceAll(`href="${file}.html"`, `href="${dir}/${file}.html"`);
|
||||
}
|
||||
|
||||
fs.writeFileSync(rootFilePath, content);
|
||||
}
|
||||
|
||||
function createChildren(files, notePath) {
|
||||
let positionCounter = 0;
|
||||
|
||||
const camelCase = name => {
|
||||
if (name === 'module-sql') {
|
||||
return 'moduleSql';
|
||||
} else if (/[^a-z]+/i.test(name)) {
|
||||
throw new Error(`Bad name '${name}'`);
|
||||
}
|
||||
|
||||
return name.charAt(0).toLowerCase() + name.substr(1);
|
||||
};
|
||||
|
||||
return files.map(file => {
|
||||
positionCounter += 10;
|
||||
|
||||
const noteId = "_" + camelCase(file);
|
||||
|
||||
return {
|
||||
"isClone": false,
|
||||
"noteId": noteId,
|
||||
"notePath": [
|
||||
...notePath,
|
||||
'_' + camelCase(file)
|
||||
],
|
||||
"title": file,
|
||||
"notePosition": positionCounter,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "html",
|
||||
"dataFileName": file + ".html"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getScriptApiMeta() {
|
||||
return [
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "_frontendApi",
|
||||
"notePath": [
|
||||
...scriptApiDocsRootNotePath,
|
||||
"_frontendApi"
|
||||
],
|
||||
"title": "API docs",
|
||||
"notePosition": 10,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "html",
|
||||
"dataFileName": "FrontendScriptApi.html",
|
||||
"dirFileName": "FrontendScriptApi",
|
||||
"children": createChildren(FE_FILES, [
|
||||
...scriptApiDocsRootNotePath,
|
||||
"_frontendApi"
|
||||
])
|
||||
},
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "_backendApi",
|
||||
"notePath": [
|
||||
...scriptApiDocsRootNotePath,
|
||||
"_backendApi"
|
||||
],
|
||||
"title": "API docs",
|
||||
"notePosition": 20,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "html",
|
||||
"dataFileName": "BackendScriptApi.html",
|
||||
"dirFileName": "BackendScriptApi",
|
||||
"children": createChildren(BE_FILES, [
|
||||
...scriptApiDocsRootNotePath,
|
||||
"_backendApi"
|
||||
])
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
@ -1,161 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
const protectedSessionService = require('../../services/protected_session');
|
||||
const utils = require('../../services/utils');
|
||||
const sql = require('../../services/sql');
|
||||
const dateUtils = require('../../services/date_utils');
|
||||
const becca = require('../becca');
|
||||
const entityChangesService = require('../../services/entity_changes');
|
||||
const AbstractBeccaEntity = require("./abstract_becca_entity");
|
||||
|
||||
/**
|
||||
* NoteAncillary represent data related/attached to the note. Conceptually similar to attributes, but intended for
|
||||
* larger amounts of data and generally not accessible to the user.
|
||||
*
|
||||
* @extends AbstractBeccaEntity
|
||||
*/
|
||||
class BNoteAncillary extends AbstractBeccaEntity {
|
||||
static get entityName() { return "note_ancillaries"; }
|
||||
static get primaryKeyName() { return "noteAncillaryId"; }
|
||||
static get hashedProperties() { return ["noteAncillaryId", "noteId", "name", "content", "utcDateModified"]; }
|
||||
|
||||
constructor(row) {
|
||||
super();
|
||||
|
||||
if (!row.noteId) {
|
||||
throw new Error("'noteId' must be given to initialize a NoteAncillary entity");
|
||||
}
|
||||
|
||||
if (!row.name) {
|
||||
throw new Error("'name' must be given to initialize a NoteAncillary entity");
|
||||
}
|
||||
|
||||
/** @type {string} needs to be set at the initialization time since it's used in the .setContent() */
|
||||
this.noteAncillaryId = row.noteAncillaryId || `${this.noteId}_${this.name}`;
|
||||
/** @type {string} */
|
||||
this.noteId = row.noteId;
|
||||
/** @type {string} */
|
||||
this.name = row.name;
|
||||
/** @type {string} */
|
||||
this.mime = row.mime;
|
||||
/** @type {boolean} */
|
||||
this.isProtected = !!row.isProtected;
|
||||
/** @type {string} */
|
||||
this.contentCheckSum = row.contentCheckSum;
|
||||
/** @type {string} */
|
||||
this.utcDateModified = row.utcDateModified;
|
||||
}
|
||||
|
||||
getNote() {
|
||||
return becca.notes[this.noteId];
|
||||
}
|
||||
|
||||
/** @returns {boolean} true if the note has string content (not binary) */
|
||||
isStringNote() {
|
||||
return utils.isStringNote(this.type, this.mime);
|
||||
}
|
||||
|
||||
/** @returns {*} */
|
||||
getContent(silentNotFoundError = false) {
|
||||
const res = sql.getRow(`SELECT content FROM note_ancillary_contents WHERE noteAncillaryId = ?`, [this.noteAncillaryId]);
|
||||
|
||||
if (!res) {
|
||||
if (silentNotFoundError) {
|
||||
return undefined;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Cannot find note ancillary content for noteAncillaryId=${this.noteAncillaryId}`);
|
||||
}
|
||||
}
|
||||
|
||||
let content = res.content;
|
||||
|
||||
if (this.isProtected) {
|
||||
if (protectedSessionService.isProtectedSessionAvailable()) {
|
||||
content = protectedSessionService.decrypt(content);
|
||||
}
|
||||
else {
|
||||
content = "";
|
||||
}
|
||||
}
|
||||
|
||||
if (this.isStringNote()) {
|
||||
return content === null
|
||||
? ""
|
||||
: content.toString("UTF-8");
|
||||
}
|
||||
else {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
setContent(content) {
|
||||
sql.transactional(() => {
|
||||
this.contentCheckSum = this.calculateCheckSum(content);
|
||||
this.save(); // also explicitly save note_ancillary to update contentCheckSum
|
||||
|
||||
const pojo = {
|
||||
noteAncillaryId: this.noteAncillaryId,
|
||||
content: content,
|
||||
utcDateModified: dateUtils.utcNowDateTime()
|
||||
};
|
||||
|
||||
if (this.isProtected) {
|
||||
if (protectedSessionService.isProtectedSessionAvailable()) {
|
||||
pojo.content = protectedSessionService.encrypt(pojo.content);
|
||||
} else {
|
||||
throw new Error(`Cannot update content of noteAncillaryId=${this.noteAncillaryId} since we're out of protected session.`);
|
||||
}
|
||||
}
|
||||
|
||||
sql.upsert("note_ancillary_contents", "noteAncillaryId", pojo);
|
||||
|
||||
entityChangesService.addEntityChange({
|
||||
entityName: 'note_ancillary_contents',
|
||||
entityId: this.noteAncillaryId,
|
||||
hash: this.contentCheckSum,
|
||||
isErased: false,
|
||||
utcDateChanged: pojo.utcDateModified,
|
||||
isSynced: true
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
calculateCheckSum(content) {
|
||||
return utils.hash(`${this.noteAncillaryId}|${content.toString()}`);
|
||||
}
|
||||
|
||||
beforeSaving() {
|
||||
if (!this.name.match(/^[a-z0-9]+$/i)) {
|
||||
throw new Error(`Name must be alphanumerical, "${this.name}" given.`);
|
||||
}
|
||||
|
||||
this.noteAncillaryId = `${this.noteId}_${this.name}`;
|
||||
|
||||
super.beforeSaving();
|
||||
|
||||
this.utcDateModified = dateUtils.utcNowDateTime();
|
||||
}
|
||||
|
||||
getPojo() {
|
||||
return {
|
||||
noteAncillaryId: this.noteAncillaryId,
|
||||
noteId: this.noteId,
|
||||
name: this.name,
|
||||
mime: this.mime,
|
||||
isProtected: !!this.isProtected,
|
||||
contentCheckSum: this.contentCheckSum,
|
||||
isDeleted: false,
|
||||
utcDateModified: this.utcDateModified
|
||||
};
|
||||
}
|
||||
|
||||
getPojoToSave() {
|
||||
const pojo = this.getPojo();
|
||||
delete pojo.content; // not getting persisted
|
||||
|
||||
return pojo;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = BNoteAncillary;
|
||||
@ -1,79 +0,0 @@
|
||||
import TypeWidget from "./type_widget.js";
|
||||
import server from "../../services/server.js";
|
||||
|
||||
const TPL = `
|
||||
<div class="note-ancillaries note-detail-printable">
|
||||
<style>
|
||||
.note-ancillaries {
|
||||
padding: 15px;
|
||||
}
|
||||
|
||||
.ancillary-content {
|
||||
max-height: 400px;
|
||||
background: var(--accented-background-color);
|
||||
padding: 10px;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.ancillary-details th {
|
||||
padding-left: 10px;
|
||||
padding-right: 10px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="alert alert-info" style="margin: 10px 0 10px 0; padding: 20px;">
|
||||
Note ancillaries are pieces of data attached to a given note, providing ancillary support.
|
||||
This view is useful for diagnostics.
|
||||
</div>
|
||||
|
||||
<div class="note-ancillary-list"></div>
|
||||
</div>`;
|
||||
|
||||
export default class AncillariesTypeWidget extends TypeWidget {
|
||||
static getType() { return "ancillaries"; }
|
||||
|
||||
doRender() {
|
||||
this.$widget = $(TPL);
|
||||
this.$list = this.$widget.find('.note-ancillary-list');
|
||||
|
||||
super.doRender();
|
||||
}
|
||||
|
||||
async doRefresh(note) {
|
||||
this.$list.empty();
|
||||
|
||||
const ancillaries = await server.get(`notes/${this.noteId}/ancillaries?includeContent=true`);
|
||||
|
||||
if (ancillaries.length === 0) {
|
||||
this.$list.html("<strong>This note has no ancillaries.</strong>");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (const ancillary of ancillaries) {
|
||||
this.$list.append(
|
||||
$('<div class="note-ancillary-wrapper">')
|
||||
.append(
|
||||
$('<h4>').append($('<span class="ancillary-name">').text(ancillary.name))
|
||||
)
|
||||
.append(
|
||||
$('<table class="ancillary-details">')
|
||||
.append(
|
||||
$('<tr>')
|
||||
.append($('<th>').text('Length:'))
|
||||
.append($('<td>').text(ancillary.contentLength))
|
||||
.append($('<th>').text('MIME:'))
|
||||
.append($('<td>').text(ancillary.mime))
|
||||
.append($('<th>').text('Date modified:'))
|
||||
.append($('<td>').text(ancillary.utcDateModified))
|
||||
)
|
||||
)
|
||||
.append(
|
||||
$('<pre class="ancillary-content">')
|
||||
.text(ancillary.content)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
import OptionsWidget from "../options_widget.js";
|
||||
|
||||
const TPL = `
|
||||
<div class="options-section">
|
||||
<h4>Extract text from PDF files</h4>
|
||||
|
||||
<label>
|
||||
<input class="extract-text-from-pdf" type="checkbox">
|
||||
Extract text from PDF
|
||||
</label>
|
||||
|
||||
<p>Text extracted from PDFs will be considered when fulltext searching.</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
export default class ExtractTextFromPdfOptions extends OptionsWidget {
|
||||
doRender() {
|
||||
this.$widget = $(TPL);
|
||||
|
||||
this.$extractTextFromPdf = this.$widget.find(".extract-text-from-pdf");
|
||||
this.$extractTextFromPdf.on("change", () =>
|
||||
this.updateCheckboxOption('extractTextFromPdf', this.$extractTextFromPdf));
|
||||
}
|
||||
|
||||
optionsLoaded(options) {
|
||||
this.setCheckboxState(this.$extractTextFromPdf, options.extractTextFromPdf);
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
import OptionsWidget from "../options_widget.js";
|
||||
|
||||
const TPL = `
|
||||
<div class="options-section">
|
||||
<h4>OCR</h4>
|
||||
|
||||
<label>
|
||||
<input class="ocr-images" type="checkbox">
|
||||
Extract text from images using OCR
|
||||
</label>
|
||||
|
||||
<p>Text extracted from images will be considered when fulltext searching.</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
export default class OcrOptions extends OptionsWidget {
|
||||
doRender() {
|
||||
this.$widget = $(TPL);
|
||||
|
||||
this.$ocrImages = this.$widget.find(".ocr-images");
|
||||
this.$ocrImages.on("change", () =>
|
||||
this.updateCheckboxOption('ocrImages', this.$ocrImages));
|
||||
}
|
||||
|
||||
optionsLoaded(options) {
|
||||
this.setCheckboxState(this.$ocrImages, options.ocrImages);
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
const protectedSession = require("./protected_session");
|
||||
const log = require("./log");
|
||||
|
||||
/**
|
||||
* @param {BNote} note
|
||||
*/
|
||||
function protectNoteAncillaries(note) {
|
||||
for (const noteAncillary of note.getNoteAncillaries()) {
|
||||
if (note.isProtected !== noteAncillary.isProtected) {
|
||||
if (!protectedSession.isProtectedSessionAvailable()) {
|
||||
log.error("Protected session is not available to fix note ancillaries.");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = noteAncillary.getContent();
|
||||
|
||||
noteAncillary.isProtected = note.isProtected;
|
||||
|
||||
// this will force de/encryption
|
||||
noteAncillary.setContent(content);
|
||||
|
||||
noteAncillary.save();
|
||||
}
|
||||
catch (e) {
|
||||
log.error(`Could not un/protect note ancillary ID = ${noteAncillary.noteAncillaryId}`);
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
protectNoteAncillaries
|
||||
}
|
||||
@ -1,150 +0,0 @@
|
||||
const Canvas = require("canvas");
|
||||
const OCRAD = require("ocrad.js");
|
||||
const log = require("./log");
|
||||
const optionService = require("./options");
|
||||
const cls = require("./cls");
|
||||
|
||||
function ocrFromByteArray(img) {
|
||||
// byte array contains raw uncompressed pixel data
|
||||
// kind: 1 - GRAYSCALE_1BPP (unsupported)
|
||||
// kind: 2 - RGB_24BPP
|
||||
// kind: 3 - RGBA_32BPP
|
||||
|
||||
if (!(img.data instanceof Uint8ClampedArray) || ![2, 3].includes(img.kind)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
const canvas = new Canvas.createCanvas(img.width, img.height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
const imageData = ctx.createImageData(img.width, img.height);
|
||||
const imageBytes = imageData.data;
|
||||
|
||||
for (let j = 0, k = 0, jj = img.width * img.height * 4; j < jj;) {
|
||||
imageBytes[j++] = img.data[k++];
|
||||
imageBytes[j++] = img.data[k++];
|
||||
imageBytes[j++] = img.data[k++];
|
||||
// in case of kind = 2, the alpha channel is missing in source pixels and we'll add it
|
||||
imageBytes[j++] = img.kind === 2 ? 255 : img.data[k++];
|
||||
}
|
||||
|
||||
ctx.putImageData(imageData, 0, 0);
|
||||
const text = OCRAD(canvas);
|
||||
|
||||
log.info(`OCR of ${img.data.length} canvas into ${text.length} chars of text took ${Date.now() - start}ms`);
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async function ocrTextFromPdfImages(pdfjsLib, page, strings) {
|
||||
const ops = await page.getOperatorList();
|
||||
|
||||
const fns = ops.fnArray;
|
||||
const args = ops.argsArray;
|
||||
|
||||
for (const arg of args) {
|
||||
const i = args.indexOf(arg);
|
||||
|
||||
if (fns[i] !== pdfjsLib.OPS.paintXObject && fns[i] !== pdfjsLib.OPS.paintImageXObject) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const imgKey = arg[0];
|
||||
const img = await new Promise((res) => page.objs.get(imgKey, r => res(r)));
|
||||
|
||||
if (!img) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const text = ocrFromByteArray(img);
|
||||
|
||||
if (text) {
|
||||
strings.push(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function extractTextFromPdf(note, buffer) {
|
||||
if (note.mime !== 'application/pdf' || !optionService.getOptionBool('extractTextFromPdf')) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const pdfjsLib = require("pdfjs-dist");
|
||||
const doc = await pdfjsLib.getDocument({data: buffer}).promise;
|
||||
let strings = [];
|
||||
|
||||
for (let p = 1; p <= doc.numPages; p++) {
|
||||
const page = await doc.getPage(p);
|
||||
|
||||
const content = await page.getTextContent({
|
||||
normalizeWhitespace: true,
|
||||
disableCombineTextItems: false
|
||||
});
|
||||
|
||||
content.items.forEach(({str}) => strings.push(str));
|
||||
|
||||
try {
|
||||
if (optionService.getOptionBool('ocrImages') && !cls.isOcrDisabled()) {
|
||||
await ocrTextFromPdfImages(pdfjsLib, page, strings);
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
log.info(`Could not OCR images from PDF note '${note.noteId}': '${e.message}', stack '${e.stack}'`);
|
||||
}
|
||||
}
|
||||
|
||||
strings = strings.filter(str => str?.trim());
|
||||
|
||||
note.saveNoteAncillary('plainText', 'text/plain', strings.join(" "));
|
||||
}
|
||||
catch (e) {
|
||||
log.info(`Extracting text from PDF on note '${note.noteId}' failed with error '${e.message}', stack ${e.stack}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function ocrTextFromBuffer(buffer) {
|
||||
// buffer is expected to contain an image in JPEG, PNG etc.
|
||||
const start = Date.now();
|
||||
|
||||
const img = await new Promise((res, rej) => {
|
||||
const img = new Canvas.Image();
|
||||
img.onload = () => res(img);
|
||||
img.onerror = err => rej(new Error("Can't load the image " + err));
|
||||
img.src = buffer;
|
||||
});
|
||||
|
||||
const canvas = new Canvas.createCanvas(img.width, img.height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(img, 0, 0, img.width, img.height);
|
||||
|
||||
const plainText = OCRAD(canvas);
|
||||
|
||||
log.info(`OCR of ${buffer.byteLength} image bytes into ${plainText.length} chars of text took ${Date.now() - start}ms`);
|
||||
return plainText;
|
||||
}
|
||||
|
||||
async function runOcr(note, buffer) {
|
||||
if (!note.isImage()
|
||||
|| !optionService.getOptionBool('ocrImages')
|
||||
|| cls.isOcrDisabled()
|
||||
|| buffer.length === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const plainText = await ocrTextFromBuffer(buffer);
|
||||
|
||||
note.saveNoteAncillary('plainText', 'text/plain', plainText);
|
||||
}
|
||||
catch (e) {
|
||||
log.error(`OCR on note '${note.noteId}' failed with error '${e.message}', stack ${e.stack}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
runOcr,
|
||||
extractTextFromPdf
|
||||
};
|
||||
@ -1,498 +0,0 @@
|
||||
"use strict"
|
||||
|
||||
const becca = require("../becca/becca");
|
||||
const fs = require("fs").promises;
|
||||
const BAttribute = require('../becca/entities/battribute');
|
||||
const utils = require('./utils');
|
||||
const log = require('./log');
|
||||
const noteService = require('./notes');
|
||||
const attributeService = require('./attributes');
|
||||
const BBranch = require('../becca/entities/bbranch');
|
||||
const path = require('path');
|
||||
const yauzl = require("yauzl");
|
||||
const htmlSanitizer = require('./html_sanitizer');
|
||||
const sql = require('./sql');
|
||||
const options = require('./options');
|
||||
const cls = require('./cls');
|
||||
const {USER_GUIDE_ZIP_DIR} = require('./resource_dir');
|
||||
|
||||
async function importUserGuideIfNeeded() {
|
||||
const userGuideSha256HashInDb = options.getOption('userGuideSha256Hash');
|
||||
let userGuideSha256HashInFile = await fs.readFile(USER_GUIDE_ZIP_DIR + "/user-guide.zip.sha256");
|
||||
|
||||
if (!userGuideSha256HashInFile || userGuideSha256HashInFile.byteLength < 64) {
|
||||
return;
|
||||
}
|
||||
|
||||
userGuideSha256HashInFile = userGuideSha256HashInFile.toString().substr(0, 64);
|
||||
|
||||
if (userGuideSha256HashInDb === userGuideSha256HashInFile) {
|
||||
// user guide ZIP file has been already imported and is up-to-date
|
||||
return;
|
||||
}
|
||||
|
||||
const hiddenRoot = becca.getNote("_hidden");
|
||||
const data = await fs.readFile(USER_GUIDE_ZIP_DIR + "/user-guide.zip", "binary");
|
||||
|
||||
cls.disableOcr(); // no OCR needed for user guide images
|
||||
|
||||
await importZip(Buffer.from(data, 'binary'), hiddenRoot);
|
||||
|
||||
options.setOption('userGuideSha256Hash', userGuideSha256HashInFile);
|
||||
}
|
||||
|
||||
async function importZip(fileBuffer, importRootNote) {
|
||||
// maps from original noteId (in ZIP file) to newly generated noteId
|
||||
const noteIdMap = {};
|
||||
const attributes = [];
|
||||
let metaFile = null;
|
||||
|
||||
function getNewNoteId(origNoteId) {
|
||||
if (origNoteId === 'root' || origNoteId.startsWith("_")) {
|
||||
// these "named" noteIds don't differ between Trilium instances
|
||||
return origNoteId;
|
||||
}
|
||||
|
||||
if (!noteIdMap[origNoteId]) {
|
||||
noteIdMap[origNoteId] = utils.newEntityId();
|
||||
}
|
||||
|
||||
return noteIdMap[origNoteId];
|
||||
}
|
||||
|
||||
function getMeta(filePath) {
|
||||
const pathSegments = filePath.split(/[\/\\]/g);
|
||||
|
||||
let cursor = {
|
||||
isImportRoot: true,
|
||||
children: metaFile.files
|
||||
};
|
||||
|
||||
let parent;
|
||||
|
||||
for (const segment of pathSegments) {
|
||||
if (!cursor || !cursor.children || cursor.children.length === 0) {
|
||||
throw new Error(`Note meta for '${filePath}' not found.`);
|
||||
}
|
||||
|
||||
parent = cursor;
|
||||
cursor = cursor.children.find(file => file.dataFileName === segment || file.dirFileName === segment);
|
||||
}
|
||||
|
||||
return {
|
||||
parentNoteMeta: parent,
|
||||
noteMeta: cursor
|
||||
};
|
||||
}
|
||||
|
||||
function getParentNoteId(filePath, parentNoteMeta) {
|
||||
return parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId);
|
||||
}
|
||||
|
||||
function getNoteId(noteMeta) {
|
||||
let userGuideNoteId;// = noteMeta.attributes?.find(attr => attr.type === 'label' && attr.name === 'helpNoteId')?.value;
|
||||
|
||||
userGuideNoteId = '_userGuide' + noteMeta.title.replace(/[^a-z0-9]/ig, '');
|
||||
|
||||
if (noteMeta.title.trim() === 'User Guide') {
|
||||
userGuideNoteId = '_userGuide';
|
||||
}
|
||||
|
||||
const noteId = userGuideNoteId || noteMeta.noteId;
|
||||
noteIdMap[noteMeta.noteId] = noteId;
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function saveAttributes(note, noteMeta) {
|
||||
if (!noteMeta) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const attr of noteMeta.attributes) {
|
||||
attr.noteId = note.noteId;
|
||||
|
||||
if (attr.type === 'label-definition') {
|
||||
attr.type = 'label';
|
||||
attr.name = `label:${attr.name}`;
|
||||
}
|
||||
else if (attr.type === 'relation-definition') {
|
||||
attr.type = 'label';
|
||||
attr.name = `relation:${attr.name}`;
|
||||
}
|
||||
|
||||
if (!attributeService.isAttributeType(attr.type)) {
|
||||
log.error(`Unrecognized attribute type ${attr.type}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(attr.name)) {
|
||||
// these relations are created automatically and as such don't need to be duplicated in the import
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attr.type === 'relation') {
|
||||
attr.value = getNewNoteId(attr.value);
|
||||
}
|
||||
|
||||
attributes.push(attr);
|
||||
}
|
||||
}
|
||||
|
||||
function saveDirectory(filePath) {
|
||||
const { parentNoteMeta, noteMeta } = getMeta(filePath);
|
||||
|
||||
const noteId = getNoteId(noteMeta);
|
||||
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
let note = becca.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
return;
|
||||
}
|
||||
|
||||
({note} = noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteMeta.title,
|
||||
content: '',
|
||||
noteId: noteId,
|
||||
type: noteMeta.type,
|
||||
mime: noteMeta.mime,
|
||||
prefix: noteMeta.prefix,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
notePosition: noteMeta.notePosition,
|
||||
isProtected: false,
|
||||
ignoreForbiddenParents: true
|
||||
}));
|
||||
|
||||
saveAttributes(note, noteMeta);
|
||||
|
||||
return noteId;
|
||||
}
|
||||
|
||||
function getNoteIdFromRelativeUrl(url, filePath) {
|
||||
while (url.startsWith("./")) {
|
||||
url = url.substr(2);
|
||||
}
|
||||
|
||||
let absUrl = path.dirname(filePath);
|
||||
|
||||
while (url.startsWith("../")) {
|
||||
absUrl = path.dirname(absUrl);
|
||||
|
||||
url = url.substr(3);
|
||||
}
|
||||
|
||||
if (absUrl === '.') {
|
||||
absUrl = '';
|
||||
}
|
||||
|
||||
absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`;
|
||||
|
||||
const {noteMeta} = getMeta(absUrl);
|
||||
const targetNoteId = getNoteId(noteMeta);
|
||||
return targetNoteId;
|
||||
}
|
||||
|
||||
function processTextNoteContent(content, filePath, noteMeta) {
|
||||
function isUrlAbsolute(url) {
|
||||
return /^(?:[a-z]+:)?\/\//i.test(url);
|
||||
}
|
||||
|
||||
content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
|
||||
if (noteMeta.title.trim() === text.trim()) {
|
||||
return ""; // remove whole H1 tag
|
||||
} else {
|
||||
return `<h2>${text}</h2>`;
|
||||
}
|
||||
});
|
||||
|
||||
content = htmlSanitizer.sanitize(content);
|
||||
|
||||
content = content.replace(/<html.*<body[^>]*>/gis, "");
|
||||
content = content.replace(/<\/body>.*<\/html>/gis, "");
|
||||
|
||||
content = content.replace(/src="([^"]*)"/g, (match, url) => {
|
||||
try {
|
||||
url = decodeURIComponent(url);
|
||||
} catch (e) {
|
||||
log.error(`Cannot parse image URL '${url}', keeping original (${e}).`);
|
||||
return `src="${url}"`;
|
||||
}
|
||||
|
||||
if (isUrlAbsolute(url) || url.startsWith("/")) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `src="api/images/${targetNoteId}/${path.basename(url)}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/href="([^"]*)"/g, (match, url) => {
|
||||
try {
|
||||
url = decodeURIComponent(url);
|
||||
} catch (e) {
|
||||
log.error(`Cannot parse link URL '${url}', keeping original (${e}).`);
|
||||
return `href="${url}"`;
|
||||
}
|
||||
|
||||
if (url.startsWith('#') // already a note path (probably)
|
||||
|| isUrlAbsolute(url)) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const targetNoteId = getNoteIdFromRelativeUrl(url, filePath);
|
||||
|
||||
return `href="#root/${targetNoteId}"`;
|
||||
});
|
||||
|
||||
content = content.replace(/data-note-path="([^"]*)"/g, (match, notePath) => {
|
||||
const noteId = notePath.split("/").pop();
|
||||
|
||||
let targetNoteId;
|
||||
|
||||
if (noteId === 'root' || noteId.startsWith("_")) { // named noteIds stay identical across instances
|
||||
targetNoteId = noteId;
|
||||
} else {
|
||||
targetNoteId = noteIdMap[noteId];
|
||||
}
|
||||
|
||||
return `data-note-path="root/${targetNoteId}"`;
|
||||
});
|
||||
|
||||
if (noteMeta) {
|
||||
const includeNoteLinks = (noteMeta.attributes || [])
|
||||
.filter(attr => attr.type === 'relation' && attr.name === 'includeNoteLink');
|
||||
|
||||
for (const link of includeNoteLinks) {
|
||||
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
|
||||
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
|
||||
}
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
function processNoteContent(noteMeta, type, mime, content, filePath) {
|
||||
if (type === 'text') {
|
||||
content = processTextNoteContent(content, filePath, noteMeta);
|
||||
}
|
||||
|
||||
if (type === 'relationMap') {
|
||||
const relationMapLinks = (noteMeta.attributes || [])
|
||||
.filter(attr => attr.type === 'relation' && attr.name === 'relationMapLink');
|
||||
|
||||
// this will replace relation map links
|
||||
for (const link of relationMapLinks) {
|
||||
// no need to escape the regexp find string since it's a noteId which doesn't contain any special characters
|
||||
content = content.replace(new RegExp(link.value, "g"), getNewNoteId(link.value));
|
||||
}
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
function saveNote(filePath, content) {
|
||||
const {parentNoteMeta, noteMeta} = getMeta(filePath);
|
||||
|
||||
if (noteMeta?.noImport) {
|
||||
return;
|
||||
}
|
||||
|
||||
const noteId = getNoteId(noteMeta);
|
||||
const parentNoteId = getParentNoteId(filePath, parentNoteMeta);
|
||||
|
||||
if (!parentNoteId) {
|
||||
throw new Error(`Cannot find parentNoteId for ${filePath}`);
|
||||
}
|
||||
|
||||
if (noteMeta?.isClone) {
|
||||
if (!becca.getBranchFromChildAndParent(noteId, parentNoteId)) {
|
||||
new BBranch({
|
||||
noteId,
|
||||
parentNoteId,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
prefix: noteMeta.prefix,
|
||||
notePosition: noteMeta.notePosition
|
||||
}).save();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let {type, mime} = noteMeta;
|
||||
|
||||
if (type !== 'file' && type !== 'image') {
|
||||
content = content.toString("UTF-8");
|
||||
}
|
||||
|
||||
content = processNoteContent(noteMeta, type, mime, content, filePath);
|
||||
|
||||
let note = becca.getNote(noteId);
|
||||
|
||||
if (note) {
|
||||
// only skeleton was created because of altered order of cloned notes in ZIP, we need to update
|
||||
// https://github.com/zadam/trilium/issues/2440
|
||||
if (note.type === undefined) {
|
||||
note.type = type;
|
||||
note.mime = mime;
|
||||
note.title = noteMeta.title;
|
||||
note.isProtected = false;
|
||||
note.save();
|
||||
}
|
||||
|
||||
note.setContent(content);
|
||||
|
||||
if (!becca.getBranchFromChildAndParent(noteId, parentNoteId)) {
|
||||
new BBranch({
|
||||
noteId,
|
||||
parentNoteId,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
prefix: noteMeta.prefix,
|
||||
notePosition: noteMeta.notePosition
|
||||
}).save();
|
||||
}
|
||||
}
|
||||
else {
|
||||
({note} = noteService.createNewNote({
|
||||
parentNoteId: parentNoteId,
|
||||
title: noteMeta.title,
|
||||
content: content,
|
||||
noteId,
|
||||
type,
|
||||
mime,
|
||||
prefix: noteMeta.prefix,
|
||||
isExpanded: noteMeta.isExpanded,
|
||||
notePosition: noteMeta.notePosition,
|
||||
isProtected: false,
|
||||
ignoreForbiddenParents: true
|
||||
}));
|
||||
|
||||
saveAttributes(note, noteMeta);
|
||||
}
|
||||
}
|
||||
|
||||
const entries = [];
|
||||
|
||||
await readZipFile(fileBuffer, async (zipfile, entry) => {
|
||||
const filePath = normalizeFilePath(entry.fileName);
|
||||
|
||||
if (/\/$/.test(entry.fileName)) {
|
||||
entries.push({
|
||||
type: 'directory',
|
||||
filePath
|
||||
});
|
||||
}
|
||||
else {
|
||||
entries.push({
|
||||
type: 'file',
|
||||
filePath,
|
||||
content: await readContent(zipfile, entry)
|
||||
});
|
||||
}
|
||||
|
||||
zipfile.readEntry();
|
||||
});
|
||||
|
||||
metaFile = JSON.parse(entries.find(entry => entry.type === 'file' && entry.filePath === '!!!meta.json').content);
|
||||
|
||||
sql.transactional(() => {
|
||||
deleteUserGuideSubtree();
|
||||
|
||||
for (const {type, filePath, content} of entries) {
|
||||
if (type === 'directory') {
|
||||
saveDirectory(filePath);
|
||||
} else if (type === 'file') {
|
||||
if (filePath === '!!!meta.json') {
|
||||
continue;
|
||||
}
|
||||
|
||||
saveNote(filePath, content);
|
||||
} else {
|
||||
throw new Error(`Unknown type ${type}`)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// we're saving attributes and links only now so that all relation and link target notes
|
||||
// are already in the database (we don't want to have "broken" relations, not even transitionally)
|
||||
for (const attr of attributes) {
|
||||
if (attr.type !== 'relation' || attr.value in becca.notes) {
|
||||
new BAttribute(attr).save();
|
||||
}
|
||||
else {
|
||||
log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is a special implementation of deleting the subtree, because we want to preserve the links to the user guide pages
|
||||
* and clones.
|
||||
*/
|
||||
function deleteUserGuideSubtree() {
|
||||
const DELETE_ID = 'user-guide';
|
||||
|
||||
function remove(branch) {
|
||||
branch.markAsDeleted(DELETE_ID);
|
||||
|
||||
const note = becca.getNote(branch.noteId);
|
||||
|
||||
for (const branch of note.getChildBranches()) {
|
||||
remove(branch);
|
||||
}
|
||||
|
||||
note.getOwnedAttributes().forEach(attr => attr.markAsDeleted(DELETE_ID));
|
||||
|
||||
note.markAsDeleted(DELETE_ID)
|
||||
}
|
||||
|
||||
remove(becca.getBranchFromChildAndParent('_userGuide', '_hidden'));
|
||||
}
|
||||
|
||||
/** @returns {string} path without leading or trailing slash and backslashes converted to forward ones */
|
||||
function normalizeFilePath(filePath) {
|
||||
filePath = filePath.replace(/\\/g, "/");
|
||||
|
||||
if (filePath.startsWith("/")) {
|
||||
filePath = filePath.substr(1);
|
||||
}
|
||||
|
||||
if (filePath.endsWith("/")) {
|
||||
filePath = filePath.substr(0, filePath.length - 1);
|
||||
}
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function streamToBuffer(stream) {
|
||||
const chunks = [];
|
||||
stream.on('data', chunk => chunks.push(chunk));
|
||||
|
||||
return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks))));
|
||||
}
|
||||
|
||||
function readContent(zipfile, entry) {
|
||||
return new Promise((res, rej) => {
|
||||
zipfile.openReadStream(entry, function(err, readStream) {
|
||||
if (err) rej(err);
|
||||
|
||||
streamToBuffer(readStream).then(res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function readZipFile(buffer, processEntryCallback) {
|
||||
return new Promise((res, rej) => {
|
||||
yauzl.fromBuffer(buffer, {lazyEntries: true, validateEntrySizes: false}, function(err, zipfile) {
|
||||
if (err) throw err;
|
||||
zipfile.readEntry();
|
||||
zipfile.on("entry", entry => processEntryCallback(zipfile, entry));
|
||||
zipfile.on("end", res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importUserGuideIfNeeded
|
||||
};
|
||||
Loading…
Reference in New Issue